Kaggle competition: Airbnb New User Bookings
<SOS> metric: NDCG@{K} Kaggle data source: airbnb-recruiting-new-user-bookings<save_to_csv>
%matplotlib inline
warnings.filterwarnings('ignore')
word2vec_pipe = Pipeline([('estimator', LogisticRegression())])
 print('F1 score: {:.3f}'.format(np.mean(cross_val_score(word2vec_pipe, X_train, y_train, scoring = 'f1'))))
word2vec_pipe.fit(X_train, y_train)
 pred = word2vec_pipe.predict(X_test)
 sample_submission.target = pred
sample_submission.to_csv('word2vec_baseline.csv', index = False )<choose_model_class>
df_train = pd.read_csv('.. /input/airbnb-recruiting-new-user-bookings/train_users_2.csv.zip')
 df_test = pd.read_csv('.. /input/airbnb-recruiting-new-user-bookings/test_users.csv.zip')
 df_train.shape, df_test.shape
param_grid = [{'estimator' : [LogisticRegression() ],
'estimator__C' : np.logspace(-3, 3, 7)},
{'estimator' : [SVC() ],
'estimator__C' : np.logspace(-1, 1, 3),
'estimator__gamma' : np.logspace(-2, 2, 5)/ X_train.shape[0]}]
word2vec_grid_search = GridSearchCV(
 estimator = word2vec_pipe,
param_grid = param_grid,
scoring = 'f1',
n_jobs = -1,
refit = True,
verbose = 1,
return_train_score = True 
)
 word2vec_grid_search.fit(X_train, y_train)
 cols = ['param_estimator',
'param_estimator__C',
'param_estimator__gamma',
'mean_test_score',
'mean_train_score']
pd.options.display.max_colwidth = 50
word2vec_grid_search_results = pd.DataFrame(word2vec_grid_search.cv_results_ ).sort_values(by = 'mean_test_score',
ascending = False)
 word2vec_grid_search_results[cols].head(10 )<save_to_csv>
labels = df_train.country_destination.values
id_test = df_test.id
df_train.drop(['country_destination'], axis = 1)
 df_all = pd.concat(( df_train, df_test), axis = 0, ignore_index= True )
pred = word2vec_grid_search.predict(X_test)
 sample_submission.target = pred
sample_submission.to_csv('word2vec_tuned.csv', index = False )<install_modules>
for i in df_all.columns:
ab = df_all[i].isnull().sum()
if ab != 0:
print(i + " has {} null values.".format(ab))
print()
!pip install transformers 
<load_from_csv>
df_all = df_all.drop(['id','country_destination','date_first_booking'], axis = 1 )
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )<categorify>
for i in df_all.columns:
ab = df_all[i].isnull().sum()
if ab != 0:
print(i + " has {} null values.".format(ab))
print()
<load_pretrained>
df_all.gender.value_counts()
tokenizer = BertTokenizer.from_pretrained('bert-large-uncased', do_lower_case=True )<import_modules>
df_all[df_all['age']<15].age = np.nan
df_all[df_all['age']>=100].age = np.nan
import numpy as np
import tensorflow as tf<categorify>
categorical_features = [
'affiliate_channel',
'affiliate_provider',
'first_affiliate_tracked',
'first_browser',
'first_device_type',
'gender',
'language',
'signup_app',
'signup_method',
'signup_flow'
]
for categorical_feature in categorical_features:
df_all[categorical_feature] = df_all[categorical_feature].astype('category' )
def bert_encode(data,maximum_length):
input_ids = []
attention_masks = []
for i in range(len(data.text)) :
encoded = tokenizer.encode_plus(
 data.text[i],
add_special_tokens=True,
max_length=maximum_length,
pad_to_max_length=True,
return_attention_mask=True, 
)
 input_ids.append(encoded['input_ids'])
 attention_masks.append(encoded['attention_mask'])
 return np.array(input_ids),np.array(attention_masks )<categorify>
df_all['date_account_created'] = pd.to_datetime(df_all['date_account_created'])
 df_all['timestamp_first_active'] = pd.to_datetime(df_all['timestamp_first_active'], format='%Y%m%d%H%M%S' )
train_input_ids,train_attention_masks = bert_encode(train,60)
 test_input_ids,test_attention_masks = bert_encode(test,60 )<choose_model_class>
sns.set_style('whitegrid')
 sns.set_context("talk" )
def create_model(bert_model):
input_ids = tf.keras.Input(shape=(60,),dtype='int32')
 attention_masks = tf.keras.Input(shape=(60,),dtype='int32')
 output = bert_model([input_ids,attention_masks])
 output = output[1]
output = tf.keras.layers.Dense(32,activation='relu' )(output)
 output = tf.keras.layers.Dropout(0.2 )(output)
 output = tf.keras.layers.Dense(1,activation='sigmoid' )(output)
 model = tf.keras.models.Model(inputs = [input_ids,attention_masks],outputs = output)
 model.compile(Adam(lr=6e-6), loss='binary_crossentropy', metrics=['accuracy'])
 return model 
<load_pretrained>
df_all['acc_year'] = df_all['date_account_created'].dt.year
df1 = df_all.groupby('acc_year' ).count()
df1.head()
years = [2010,2011,2012,2013,2014]
yearsOrder=pd.api.types.CategoricalDtype(ordered=True, categories=years)
 df1.reset_index(inplace = True)
 df1.acc_year = df1.acc_year.astype(yearsOrder )
bert_model = TFBertModel.from_pretrained('bert-large-uncased' )<train_model>
df2 = df_all[df_all['date_account_created'].dt.year==2014]
df2['monthYear14'] = df2['date_account_created'].map(lambda x: x.strftime('%m-%Y'))
df2 = df2.groupby('monthYear14' ).count()
history = model.fit([train_input_ids,train_attention_masks],train.target,validation_split=0.2, epochs=2,batch_size=10 )<predict_on_test>
df_all['member_age_bins']=df_all['age'].apply(lambda x: '18 - 20' if 18<x<=20
else '20 - 30' if 20<x<=30
else '30 - 40' if 30<x<=40
else '40 - 50' if 40<x<=50
else '50 - 60' if 50<x<=60
else '60-70' if 60<x<=70
else '70+' if 70<x<=100
else np.nan )
result = model.predict([test_input_ids,test_attention_masks])
 result = np.round(result ).astype(int )<save_to_csv>
df3 = df_all[df_all['date_account_created'].dt.year==2013]
df3['monthYear13'] = df3['date_account_created'].map(lambda x: x.strftime('%m-%Y'))
df3 = df3.groupby('monthYear13' ).count()
result = pd.DataFrame(result)
 submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 output = pd.DataFrame({'id':submission.id,'target':result[0]})
 output.to_csv('submission.csv',index=False )<set_options>
df3 = df_all[df_all['date_account_created'].dt.year==2013]
df3['monthYear13'] = df3['date_account_created'].map(lambda x: x.strftime('%m-%Y'))
df3= df3.groupby(['monthYear13','member_age_bins'] ).count()
df3.reset_index(inplace=True )
warnings.filterwarnings('ignore' )<randomize_order>
df3 = df_all[df_all['date_account_created'].dt.year==2013]
df3['monthYear13'] = df3['date_account_created'].map(lambda x: x.strftime('%m-%Y'))
df3= df3.groupby(['monthYear13','gender'] ).count()
df3.reset_index(inplace=True )
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 def df_parallelize_run(func, t_split):
num_cores = np.min([N_CORES,len(t_split)])
 pool = Pool(num_cores)
 df = pd.concat(pool.map(func, t_split), axis=1)
 pool.close()
pool.join()
return df<categorify>
df_train = df_train.drop(['country_destination'], axis=1 )
def get_data_by_store(store):
df = pd.concat([pd.read_pickle(BASE),
pd.read_pickle(PRICE ).iloc[:,2:],
pd.read_pickle(CALENDAR ).iloc[:,2:]],
axis=1)
 df = df[df['store_id']==store]
df2 = pd.read_pickle(MEAN_ENC)[mean_features]
df2 = df2[df2.index.isin(df.index)]
df3 = pd.read_pickle(LAGS ).iloc[:,3:]
df3 = df3[df3.index.isin(df.index)]
df = pd.concat([df, df2], axis=1)
 del df2
df = pd.concat([df, df3], axis=1)
 del df3
features = [col for col in list(df)if col not in remove_features]
df = df[['id','d',TARGET]+features]
df = df[df['d']>=START_TRAIN].reset_index(drop=True)
 return df, features
def get_base_test() :
base_test = pd.DataFrame()
for store_id in STORES_IDS:
temp_df = pd.read_pickle(AUX_MODELS+'test_'+store_id+'.pkl')
 temp_df['store_id'] = store_id
base_test = pd.concat([base_test, temp_df] ).reset_index(drop=True)
 return base_test
def make_lag(LAG_DAY):
lag_df = base_test[['id','d',TARGET]]
col_name = 'sales_lag_'+str(LAG_DAY)
 lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(LAG_DAY)).astype(np.float16)
 return lag_df[[col_name]]
def make_lag_roll(LAG_DAY):
shift_day = LAG_DAY[0]
roll_wind = LAG_DAY[1]
lag_df = base_test[['id','d',TARGET]]
col_name = 'rolling_mean_tmp_'+str(shift_day)+'_'+str(roll_wind)
 lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(shift_day ).rolling(roll_wind ).mean())
 return lag_df[[col_name]]<init_hyperparams>
dfWork = pd.concat(( df_train, df_test), axis=0, ignore_index=True)
 dfWork = dfWork.drop(['id', 'date_first_booking'], axis=1)
 dfWork = dfWork.fillna(-1)
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'tweedie',
'tweedie_variance_power': 1.1,
'metric': 'rmse',
'subsample': 0.5,
'subsample_freq': 1,
'learning_rate': 0.03,
'num_leaves': 2**11-1,
'min_data_in_leaf': 2**12-1,
'feature_fraction': 0.5,
'max_bin': 100,
'n_estimators': 1400,
'boost_from_average': False,
'verbose': 1,
} 
<define_variables>
dac = np.vstack(dfWork.date_account_created.astype(str ).apply(lambda x: list(map(int, x.split('-')))).values)
 dfWork['dac_year'] = dac[:,0]
dfWork['dac_month'] = dac[:,1]
dfWork['dac_day'] = dac[:,2]
dfWork = dfWork.drop(['date_account_created'], axis=1 )
SEED = 42
seed_everything(SEED)
 lgb_params['seed'] = SEED
N_CORES = psutil.cpu_count()
TARGET = 'sales'
START_TRAIN = 0
END_TRAIN = 1913
P_HORIZON = 28
USE_AUX = True
remove_features = ['id','state_id','store_id',
'date','wm_yr_wk','d',TARGET]
mean_features = ['enc_cat_id_mean','enc_cat_id_std',
'enc_dept_id_mean','enc_dept_id_std',
'enc_item_id_mean','enc_item_id_std']
ORIGINAL = '.. /input/m5-forecasting-accuracy/'
BASE = '.. /input/m5-simple-fe/grid_part_1.pkl'
PRICE = '.. /input/m5-simple-fe/grid_part_2.pkl'
CALENDAR = '.. /input/m5-simple-fe/grid_part_3.pkl'
LAGS = '.. /input/m5-lags-features/lags_df_28.pkl'
MEAN_ENC = '.. /input/m5-custom-features/mean_encoding_df.pkl'
AUX_MODELS = '.. /input/m5groupkfold/'
STORES_IDS = ['CA_1']
CV_FOLDS = [0,1,2]
SHIFT_DAY = 28
N_LAGS = 15
LAGS_SPLIT = [col for col in range(SHIFT_DAY,SHIFT_DAY+N_LAGS)]
ROLS_SPLIT = []
for i in [1,7,14]:
for j in [7,14,30,60]:
ROLS_SPLIT.append([i,j] )<init_hyperparams>
tfa = np.vstack(dfWork.timestamp_first_active.astype(str ).apply(lambda x: list(map(int, [x[:4],x[4:6],x[6:8],x[8:10],x[10:12],x[12:14]])) ).values)
 dfWork['tfa_year'] = tfa[:,0]
dfWork['tfa_month'] = tfa[:,1]
dfWork['tfa_day'] = tfa[:,2]
dfWork = dfWork.drop(['timestamp_first_active'], axis=1 )
for store_id in STORES_IDS:
print('Train', store_id)
 grid_df, features_columns = get_data_by_store(store_id)
 train_mask = grid_df['d']<=END_TRAIN
preds_mask = grid_df['d']>(END_TRAIN-100)
 folds = GroupKFold(n_splits=3)
 grid_df['groups'] = grid_df['tm_w'].astype(str)+ '_' + grid_df['tm_y'].astype(str)
 split_groups = grid_df[train_mask]['groups']
X,y = grid_df[train_mask][features_columns], grid_df[train_mask][TARGET]
grid_df = grid_df[preds_mask].reset_index(drop=True)
 keep_cols = [col for col in list(grid_df)if '_tmp_' not in col]
grid_df = grid_df[keep_cols]
grid_df.to_pickle('test_'+store_id+'.pkl')
 del grid_df
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y, groups=split_groups)) :
print('Fold:',fold_)
 print(len(trn_idx),len(val_idx))
tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
v_X, v_y = X.iloc[val_idx,:], y[val_idx]
train_data = lgb.Dataset(tr_x, label=tr_y)
 valid_data = lgb.Dataset(v_X, label=v_y)
 seed_everything(SEED)
 estimator = lgb.train(
 lgb_params,
train_data,
valid_sets = [train_data, valid_data],
verbose_eval = 100 
)
 model_name = 'lgb_model_'+STORES_IDS[0]+'_'+str(fold_)+'.bin'
pickle.dump(estimator, open(model_name, 'wb'))
del train_data, valid_data, estimator
gc.collect()
MODEL_FEATURES = features_columns<define_variables>
av = dfWork.age.values
dfWork['age'] = np.where(np.logical_or(av<14, av>100), -1, av )
for fold_ in CV_FOLDS:
print("FOLD:", fold_)
 all_preds = pd.DataFrame()
base_test = get_base_test()
main_time = time.time()
for PREDICT_DAY in range(1,29):
print('Predict | Day:', PREDICT_DAY)
 start_time = time.time()
grid_df = base_test.copy()
grid_df = pd.concat([grid_df, df_parallelize_run(make_lag_roll, ROLS_SPLIT)], axis=1)
 for store_id in STORES_IDS:
model_path = 'lgb_model_'+store_id+'_'+str(fold_)+'.bin'
if USE_AUX:
model_path = AUX_MODELS + model_path
estimator = pickle.load(open(model_path, 'rb'))
day_mask = base_test['d']==(END_TRAIN+PREDICT_DAY)
 store_mask = base_test['store_id']==store_id
mask =(day_mask)&(store_mask)
 base_test[TARGET][mask] = estimator.predict(grid_df[mask][MODEL_FEATURES])
 temp_df = base_test[day_mask][['id',TARGET]]
temp_df.columns = ['id','F'+str(PREDICT_DAY)]
if 'id' in list(all_preds):
all_preds = all_preds.merge(temp_df, on=['id'], how='left')
 else:
all_preds = temp_df.copy()
all_preds = all_preds.reset_index(drop=True)
 print('
' %0.2f min total |' %(( time.time() - main_time)/ 60),
' %0.2f day sales |' %(temp_df['F'+str(PREDICT_DAY)].sum()))
all_preds.to_csv('all_preds_'+str(fold_)+'.csv',index=False)
 del temp_df, all_preds<load_from_csv>
ohe_feats = ['gender', 'signup_method', 'signup_flow', 'language', 'affiliate_channel',
'affiliate_provider', 'first_affiliate_tracked', 'signup_app', 'first_device_type', 'first_browser']
for f in ohe_feats:
dfWork_dummy = pd.get_dummies(dfWork[f], prefix=f)
 dfWork = dfWork.drop([f], axis=1)
 dfWork = pd.concat(( dfWork, dfWork_dummy), axis=1 )
all_preds_0=pd.read_csv(AUX_MODELS+'all_preds_0'+'.csv')
 all_preds_1=pd.read_csv(AUX_MODELS+'all_preds_1'+'.csv')
 all_preds_2=pd.read_csv(AUX_MODELS+'all_preds_2'+'.csv' )<create_dataframe>
vals = dfWork.values
X = vals[:df_train.shape[0]]
le = LabelEncoder()
y = le.fit_transform(labels)
 X_test = vals[df_train.shape[0]:]
final_all_preds = pd.DataFrame()
final_all_preds['id'] = all_preds_1['id']
for item in all_preds_1:
if item!='id':
final_all_preds[item]=(all_preds_0[item]*(1/3)) +(all_preds_1[item]*(1/3)) +(all_preds_2[item]*(1/3))
final_all_preds<save_to_csv>
xgb = XGBClassifier(max_depth=6, learning_rate=0.3, n_estimators=25,
objective='multi:softprob', subsample=0.5, colsample_bytree=0.5, seed=0)
 xgb.fit(X, y )
submission = pd.read_csv(ORIGINAL+'sample_submission.csv')[['id']]
submission = submission.merge(final_all_preds, on=['id'], how='left' ).fillna(0)
 submission.to_csv('submission'+'.csv', index=False )<set_options>
y_pred = xgb.predict_proba(X_test )
warnings.filterwarnings('ignore' )<randomize_order>
ids = []
cts = []
for i in range(len(id_test)) :
idx = id_test[i]
ids += [idx] * 5
cts += le.inverse_transform(np.argsort(y_pred[i])[::-1])[:5].tolist()
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 def df_parallelize_run(func, t_split):
num_cores = np.min([N_CORES,len(t_split)])
 pool = Pool(num_cores)
 df = pd.concat(pool.map(func, t_split), axis=1)
 pool.close()
pool.join()
return df<load_pretrained>
sub = pd.DataFrame(np.column_stack(( ids, cts)) , columns=['id', 'country'])
 sub.to_csv('sub.csv',index=False )