Kaggle competition: Predict Future Sales
pred=model.predict(test2 )<save_to_csv>
M = pd.read_pickle('/kaggle/input/sales-data-prep/matrix.pkl')
 M.columns
my_submission = pd.DataFrame({'Id': test.Id, 'Target': pred})
 my_submission.to_csv('submission.csv', index=False )<set_options>
M = pd.read_pickle('/kaggle/input/sales-data-prep/matrix.pkl')
 M.drop(["new_item_cat_enc_lag_1", "new_item_cat_enc_lag_2", "new_item_cat_enc_lag_3"], axis=1, inplace=True)
 M = M[M["date_block_num"] > 2]
M.fillna(0)
 for col in M.columns:
print(col,M[col].nunique())
 def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
X_train = M[M.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = M[M.date_block_num < 33]['item_cnt_month']
X_test = M[M.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 X_val = M[M.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_val = M[M.date_block_num == 33]['item_cnt_month']
del M
gc.collect()
warnings.filterwarnings("ignore")
<save_to_csv>
model = XGBRegressor(
 max_depth=10,
booster='gbtree',
n_estimators=1000,
min_child_weight=0.5,
subsample=0.8,
sampling_method="uniform",
colsample_bynode=1,
colsample_bytree=0.8,
eta=0.1,
tree_method='gpu_hist',
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_val, Y_val)],
verbose=True,
early_stopping_rounds = 20 )
df = pd.read_csv('.. /input/sample_submission.csv')
 for i in np.arange(1,5):
df['Target'] = i
df.to_csv('sample_{}.csv'.format(i), index = False )<load_from_csv>
pickle.dump(model, open("model.pkl", "wb"))
train_df = pd.read_csv('.. /input/train.csv')
 train_scored = train_df.query('parentesco1 == 1')
 train_dist = train_scored['Target'].value_counts(normalize = True ).tolist()
train_dist = [train_dist[3],train_dist[1],train_dist[2],train_dist[0]]
print('Train distributions are', train_dist)
 difference = target_dist - train_dist
print('The difference in the test and target distributions is', difference )<load_from_csv>
loaded_model = pickle.load(open("model.pkl", "rb"))
train_set = pd.read_csv('.. /input/train.csv')
 test_set = pd.read_csv('.. /input/test.csv')
 test_set.loc[test_set['rez_esc'] == 99.0 , 'rez_esc'] = 5
def repalce_v18q1(x):
if x['v18q'] == 0:
return x['v18q']
else:
return x['v18q1']
train_set['v18q1'] = train_set.apply(lambda x : repalce_v18q1(x),axis=1)
 test_set['v18q1'] = test_set.apply(lambda x : repalce_v18q1(x),axis=1)
 train_set['v2a1'] = train_set['v2a1'].fillna(value=train_set['tipovivi3'])
 test_set['v2a1'] = test_set['v2a1'].fillna(value=test_set['tipovivi3'])
 cols = ['edjefe', 'edjefa', 'dependency']
train_set[cols] = train_set[cols].replace({'no': 0, 'yes':1} ).astype(float)
 test_set[cols] = test_set[cols].replace({'no': 0, 'yes':1} ).astype(float)
 train_set['roof_waste_material'] = np.nan
test_set['roof_waste_material'] = np.nan
train_set['electricity_other'] = np.nan
test_set['electricity_other'] = np.nan
def fill_roof_exception(x):
if(x['techozinc'] == 0)and(x['techoentrepiso'] == 0)and(x['techocane'] == 0)and(x['techootro'] == 0):
return 1
else:
return 0
def fill_no_electricity(x):
if(x['public'] == 0)and(x['planpri'] == 0)and(x['noelec'] == 0)and(x['coopele'] == 0):
return 1
else:
return 0
train_set['roof_waste_material'] = train_set.apply(lambda x : fill_roof_exception(x),axis=1)
 test_set['roof_waste_material'] = test_set.apply(lambda x : fill_roof_exception(x),axis=1)
 train_set['electricity_other'] = train_set.apply(lambda x : fill_no_electricity(x),axis=1)
 test_set['electricity_other'] = test_set.apply(lambda x : fill_no_electricity(x),axis=1)
 def owner_is_adult(x):
if x['age'] <= 18:
return 0
else:
return 1
train_set['head_less_18'] = train_set.apply(lambda x : owner_is_adult(x),axis=1)
 test_set['head_less_18'] = test_set.apply(lambda x : owner_is_adult(x),axis=1)
 train_set['adult'] = train_set['hogar_adul'] - train_set['hogar_mayor']
train_set['dependency_count'] = train_set['hogar_nin'] + train_set['hogar_mayor']
train_set['dependency'] = train_set['dependency_count'] / train_set['adult']
train_set['child_percent'] = train_set['hogar_nin']/train_set['hogar_total']
train_set['elder_percent'] = train_set['hogar_mayor']/train_set['hogar_total']
train_set['adult_percent'] = train_set['hogar_adul']/train_set['hogar_total']
test_set['adult'] = test_set['hogar_adul'] - test_set['hogar_mayor']
test_set['dependency_count'] = test_set['hogar_nin'] + test_set['hogar_mayor']
test_set['dependency'] = test_set['dependency_count'] / test_set['adult']
test_set['child_percent'] = test_set['hogar_nin']/test_set['hogar_total']
test_set['elder_percent'] = test_set['hogar_mayor']/test_set['hogar_total']
test_set['adult_percent'] = test_set['hogar_adul']/test_set['hogar_total']
train_set['rent_per_adult'] = train_set['v2a1']/(train_set['hogar_adul']+0.1)
 train_set['rent_per_person'] = train_set['v2a1']/train_set['hhsize']
test_set['rent_per_adult'] = test_set['v2a1']/(test_set['hogar_adul']+0.1)
 test_set['rent_per_person'] = test_set['v2a1']/test_set['hhsize']
train_set['overcrowding_room_and_bedroom'] =(train_set['hacdor'] + train_set['hacapo'])/2
test_set['overcrowding_room_and_bedroom'] =(test_set['hacdor'] + test_set['hacapo'])/2
train_set['no_appliances'] = train_set['refrig'] + train_set['computer'] + train_set['television']
test_set['no_appliances'] = test_set['refrig'] + test_set['computer'] + test_set['television']
train_set['r4h1_percent_in_male'] = train_set['r4h1'] / train_set['r4h3']
train_set['r4m1_percent_in_female'] = train_set['r4m1'] / train_set['r4m3']
train_set['r4h1_percent_in_total'] = train_set['r4h1'] / train_set['hhsize']
train_set['r4m1_percent_in_total'] = train_set['r4m1'] / train_set['hhsize']
train_set['r4t1_percent_in_total'] = train_set['r4t1'] / train_set['hhsize']
test_set['r4h1_percent_in_male'] = test_set['r4h1'] / test_set['r4h3']
test_set['r4m1_percent_in_female'] = test_set['r4m1'] / test_set['r4m3']
test_set['r4h1_percent_in_total'] = test_set['r4h1'] / test_set['hhsize']
test_set['r4m1_percent_in_total'] = test_set['r4m1'] / test_set['hhsize']
test_set['r4t1_percent_in_total'] = test_set['r4t1'] / test_set['hhsize']
train_set['rent_per_room'] = train_set['v2a1']/train_set['rooms']
train_set['bedroom_per_room'] = train_set['bedrooms']/train_set['rooms']
train_set['elder_per_room'] = train_set['hogar_mayor']/train_set['rooms']
train_set['adults_per_room'] = train_set['adult']/train_set['rooms']
train_set['child_per_room'] = train_set['hogar_nin']/train_set['rooms']
train_set['male_per_room'] = train_set['r4h3']/train_set['rooms']
train_set['female_per_room'] = train_set['r4m3']/train_set['rooms']
train_set['room_per_person_household'] = train_set['hhsize']/train_set['rooms']
test_set['rent_per_room'] = test_set['v2a1']/test_set['rooms']
test_set['bedroom_per_room'] = test_set['bedrooms']/test_set['rooms']
test_set['elder_per_room'] = test_set['hogar_mayor']/test_set['rooms']
test_set['adults_per_room'] = test_set['adult']/test_set['rooms']
test_set['child_per_room'] = test_set['hogar_nin']/test_set['rooms']
test_set['male_per_room'] = test_set['r4h3']/test_set['rooms']
test_set['female_per_room'] = test_set['r4m3']/test_set['rooms']
test_set['room_per_person_household'] = test_set['hhsize']/test_set['rooms']
train_set['rent_per_bedroom'] = train_set['v2a1']/train_set['bedrooms']
train_set['edler_per_bedroom'] = train_set['hogar_mayor']/train_set['bedrooms']
train_set['adults_per_bedroom'] = train_set['adult']/train_set['bedrooms']
train_set['child_per_bedroom'] = train_set['hogar_nin']/train_set['bedrooms']
train_set['male_per_bedroom'] = train_set['r4h3']/train_set['bedrooms']
train_set['female_per_bedroom'] = train_set['r4m3']/train_set['bedrooms']
train_set['bedrooms_per_person_household'] = train_set['hhsize']/train_set['bedrooms']
test_set['rent_per_bedroom'] = test_set['v2a1']/test_set['bedrooms']
test_set['edler_per_bedroom'] = test_set['hogar_mayor']/test_set['bedrooms']
test_set['adults_per_bedroom'] = test_set['adult']/test_set['bedrooms']
test_set['child_per_bedroom'] = test_set['hogar_nin']/test_set['bedrooms']
test_set['male_per_bedroom'] = test_set['r4h3']/test_set['bedrooms']
test_set['female_per_bedroom'] = test_set['r4m3']/test_set['bedrooms']
test_set['bedrooms_per_person_household'] = test_set['hhsize']/test_set['bedrooms']
train_set['tablet_per_person_household'] = train_set['v18q1']/train_set['hhsize']
train_set['phone_per_person_household'] = train_set['qmobilephone']/train_set['hhsize']
test_set['tablet_per_person_household'] = test_set['v18q1']/test_set['hhsize']
test_set['phone_per_person_household'] = test_set['qmobilephone']/test_set['hhsize']
train_set['age_12_19'] = train_set['hogar_nin'] - train_set['r4t1']
test_set['age_12_19'] = test_set['hogar_nin'] - test_set['r4t1']
train_set['escolari_age'] = train_set['escolari']/train_set['age']
test_set['escolari_age'] = test_set['escolari']/test_set['age']
train_set['rez_esc_escolari'] = train_set['rez_esc']/(train_set['escolari']+0.1)
 train_set['rez_esc_r4t1'] = train_set['rez_esc']/(train_set['r4t1']+0.1)
 train_set['rez_esc_r4t2'] = train_set['rez_esc']/train_set['r4t2']
train_set['rez_esc_r4t3'] = train_set['rez_esc']/train_set['r4t3']
train_set['rez_esc_age'] = train_set['rez_esc']/train_set['age']
test_set['rez_esc_escolari'] = test_set['rez_esc']/(test_set['escolari']+0.1)
 test_set['rez_esc_r4t1'] = test_set['rez_esc']/(test_set['r4t1']+0.1)
 test_set['rez_esc_r4t2'] = test_set['rez_esc']/test_set['r4t2']
test_set['rez_esc_r4t3'] = test_set['rez_esc']/test_set['r4t3']
test_set['rez_esc_age'] = test_set['rez_esc']/test_set['age']
train_set['dependency'] = train_set['dependency'].replace({np.inf: 0})
 test_set['dependency'] = test_set['dependency'].replace({np.inf: 0})
 df_train = pd.DataFrame()
df_test = pd.DataFrame()
aggr_mean_list = ['rez_esc', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7', 'parentesco2',
'parentesco3', 'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10', 'parentesco11', 'parentesco12',
'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9',]
other_list = ['escolari', 'age', 'escolari_age']
for item in aggr_mean_list:
group_train_mean = train_set[item].groupby(train_set['idhogar'] ).mean()
group_test_mean = test_set[item].groupby(test_set['idhogar'] ).mean()
new_col = item + '_aggr_mean'
df_train[new_col] = group_train_mean
df_test[new_col] = group_test_mean
for item in other_list:
for function in ['mean','std','min','max','sum']:
group_train = train_set[item].groupby(train_set['idhogar'] ).agg(function)
 group_test = test_set[item].groupby(test_set['idhogar'] ).agg(function)
 new_col = item + '_' + function
df_train[new_col] = group_train
df_test[new_col] = group_test
df_test = df_test.reset_index()
df_train = df_train.reset_index()
train_agg = pd.merge(train_set, df_train, on='idhogar')
 test = pd.merge(test_set, df_test, on='idhogar')
 train_agg.fillna(value=0, inplace=True)
 test.fillna(value=0, inplace=True)
 train = train_agg.query('parentesco1==1')
 submission = test[['Id']]
test = test.query('parentesco1==1')
 pred_train_dist = test[['Id']]
pred_test_dist = test[['Id']]
train.drop(columns=['idhogar','Id', 'tamhog', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace=True)
 test.drop(columns=['idhogar','Id', 'tamhog', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace=True)
 print('Train shape:',train.shape)
 print('Test shape:', test.shape)
 y = train['Target']
train.drop(columns=['Target'], inplace=True)
 scaler = MinMaxScaler()
train = scaler.fit_transform(train)
 test = scaler.transform(test)
 def classifyrank(data, cutoff):
data = rankdata(data)
 length = data.size
clas = []
for j in data:
i = j/length
if i < cutoff[0]:
clas.append(1)
 elif i < cutoff[0]+cutoff[1]:
clas.append(2)
 elif i < cutoff[0]+cutoff[1]+cutoff[2]:
clas.append(3)
 else:
clas.append(4)
 return clas<find_best_model_class>
test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 Y_test = loaded_model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
} )
clf = Ridge(alpha = 5)
 kfold = 5
kf = StratifiedKFold(n_splits=kfold, shuffle=True, random_state = 9)
 predicts_result = []
ave_score = 0
for train_index, test_index in kf.split(train, y):
X_train, X_val = train[train_index], train[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
clf.fit(X_train, y_train)
 predicts_result.append(clf.predict(test))
ave_score += f1_score(y_val, classifyrank(clf.predict(X_val), train_dist), average = 'macro')/kfold
print('Cross Validation Macro F1 score:',ave_score )<save_to_csv>
items=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/items.csv")
 item_categories=pd.read_csv("/kaggle/input/competitive-data-science-predict-future-sales/item_categories.csv" )
predicts_result = rankdata(np.array(predicts_result ).mean(axis = 0))
target_train_dist = classifyrank(predicts_result, train_dist)
 target_test_dist = classifyrank(predicts_result, target_dist)
 pred_train_dist['Target'] = target_train_dist
pred_test_dist['Target'] = target_test_dist
print('Using Train Distribution:')
 print(pred_train_dist['Target'].value_counts())
 print('
Using Test Distribution:')
 print(pred_test_dist['Target'].value_counts())
 train_submission = submission.copy()
train_submission = train_submission.merge(pred_train_dist[['Id', 'Target']], on = 'Id', how = 'left')
 train_submission = train_submission.fillna(4)
 train_submission['Target'] = train_submission['Target'].astype(int)
 train_submission.to_csv('train_dist.csv', index = False)
 test_submission = submission.copy()
test_submission = test_submission.merge(pred_test_dist[['Id', 'Target']], on = 'Id', how = 'left')
 test_submission = test_submission.fillna(4)
 test_submission['Target'] = test_submission['Target'].astype(int)
 test_submission.to_csv('test_dist.csv', index = False )<set_options>
df = pd.merge(items, item_categories)
 df
nltk.download('punkt')
 def seed_everything(seed=0):
np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
SEED = 40
seed_everything(SEED )<categorify>
submission.to_csv('my_submission.csv', index=False )