Kaggle competition: Recruit Restaurant Visitor Forecasting
<SOS> metric: RMSLE Kaggle data source: recruit-restaurant-visitor-forecasting<train_on_grid>
%matplotlib inline
numberbatch_matrix, unknown_words_numberbatch = build_matrix(tokenizer.word_index, NUMBERBATCH_EMBEDDING_PATH)
 print('n unknown words(numberbatch): ', len(unknown_words_numberbatch))
embedding_matrix = numberbatch_matrix
print(embedding_matrix.shape)
 del numberbatch_matrix
gc.collect()<find_best_model_class>
def LeaveOneOut(data1, data2, groupcolumns, columnName, useLOO=False, cut=1, addNoise=False):
features = list([])
 for a in groupcolumns:
features.append(a)
 if(columnName is not None):
features.append(columnName)
 grpCount = data1.groupby(features)['visitors'].count().reset_index().rename(columns={'visitors': 'Count'})
 if(useLOO):
grpCount = grpCount[(grpCount.Count > cut)]
grpMean = data1.groupby(features)['visitors'].mean().reset_index().rename(columns={'visitors': 'Mean'})
 grpMedian = data1.groupby(features)['visitors'].median().reset_index().rename(columns={'visitors': 'Median'})
 grpMin = data1.groupby(features)['visitors'].min().reset_index().rename(columns={'visitors': 'Min'})
 grpMax = data1.groupby(features)['visitors'].max().reset_index().rename(columns={'visitors': 'Max'})
 grpStd = data1.groupby(features)['visitors'].std().reset_index().rename(columns={'visitors': 'Std'})
 grpOutcomes = grpCount.merge(grpMean, on=features)
 grpOutcomes = grpOutcomes.merge(grpMedian, on=features)
 grpOutcomes = grpOutcomes.merge(grpMin, on=features)
 grpOutcomes = grpOutcomes.merge(grpMax, on=features)
 grpOutcomes = grpOutcomes.merge(grpStd, on=features)
 x = pd.merge(data2[features], grpOutcomes,
suffixes=('x_', ''),
how='left',
on=features,
left_index=True)[['Count','Mean','Median','Max','Min','Std']]
x['Outcomes'] = data2['visitors'].values
if(useLOO):
nonnulls =(~x.Count.isnull())
 x.loc[nonnulls,'Mean'] =(( x[nonnulls].Mean*x[nonnulls].Count)-x[nonnulls].Outcomes)
 x.loc[nonnulls,'Median'] =(( x[nonnulls].Median*x[nonnulls].Count)-x[nonnulls].Outcomes)
 if(addNoise is True):
x.loc[nonnulls&(x.Std>0),'Mean'] += np.random.normal(0,x[nonnulls&(x.Std>0)].Std,x[nonnulls&(x.Std>0)].shape[0])
 x.loc[nonnulls&(x.Std>0),'Median'] += np.random.normal(0,x[nonnulls&(x.Std>0)].Std,x[nonnulls&(x.Std>0)].shape[0])
 else:
x.loc[nonnulls,'Count'] -= 1
x.loc[nonnulls,'Mean'] /=(x[nonnulls].Count)
 x.loc[nonnulls,'Median'] /=(x[nonnulls].Count)
 x.Count = np.log1p(x.Count)
 x = x.replace(np.inf, np.nan)
 x = x.replace(-np.inf, np.nan)
 return x[['Count','Mean','Median','Max','Min', 'Std']]
def MungeTrain() :
air_visit_data = pd.read_csv('.. /input/air_visit_data.csv',parse_dates=['visit_date'])
 air_store_info = pd.read_csv('.. /input/air_store_info.csv')
 hpg_store_info = pd.read_csv('.. /input/hpg_store_info.csv')
 hpg_store_info.drop(['latitude','longitude'],inplace=True,axis=1)
 store_id_relation = pd.read_csv('.. /input/store_id_relation.csv')
 storeinfo = air_store_info.merge(store_id_relation,on='air_store_id',how='left')
 storeinfo = storeinfo.merge(hpg_store_info,on='hpg_store_id',how='left')
 air_reserve = pd.read_csv('.. /input/air_reserve.csv',parse_dates=['visit_datetime'])
 air_reserve['visit_date'] = air_reserve.visit_datetime.apply(lambda df :
datetime.datetime(year=df.year, month=df.month, day=df.day))
hpg_reserve =pd.read_csv('.. /input/hpg_reserve.csv',parse_dates=['visit_datetime'])
 hpg_reserve['visit_date'] = hpg_reserve.visit_datetime.apply(lambda df :
datetime.datetime(year=df.year, month=df.month, day=df.day))
date_info = pd.read_csv('.. /input/date_info.csv',parse_dates=['calendar_date'] ).rename(columns={'calendar_date':'visit_date'})
 air_reserve_by_date = air_reserve.groupby(['air_store_id','visit_date'] ).reserve_visitors.sum().reset_index(drop=False)
 hpg_reserve_by_date = hpg_reserve.groupby(['hpg_store_id','visit_date'] ).reserve_visitors.sum().reset_index(drop=False)
 train = air_visit_data.merge(storeinfo,on='air_store_id',how='left')
 train = train.merge(air_reserve_by_date,on=['air_store_id','visit_date'],how='left')
 train = train.merge(hpg_reserve_by_date,on=['hpg_store_id','visit_date'],how='left')
 train = train.merge(date_info,on='visit_date',how='left')
 train['year'] = train.visit_date.dt.year
train['month'] = train.visit_date.dt.month
train.reserve_visitors_x = train.reserve_visitors_x.fillna(0)
 train.reserve_visitors_y = train.reserve_visitors_y.fillna(0)
 train.reserve_visitors_x = np.log1p(train.reserve_visitors_x)
 train.reserve_visitors_y = np.log1p(train.reserve_visitors_y)
 train.visitors = np.log1p(train.visitors)
 train = train.fillna(-1)
 train = train.sort_values(by=['visit_date','air_store_id'],ascending=False)
 train = train.reset_index(drop=True)
 return train
def MungeTest(columns):
air_visit_data = pd.read_csv('.. /input/sample_submission.csv')
 air_visit_data['visit_date'] = air_visit_data.id.apply(lambda x: datetime.datetime(year=int(x[-10:-6]), month=int(x[-5:-3]), day=int(x[-2:])))
 air_visit_data['air_store_id'] = air_visit_data.id.apply(lambda x: x[:-11])
 air_store_info = pd.read_csv('.. /input/air_store_info.csv')
 hpg_store_info = pd.read_csv('.. /input/hpg_store_info.csv')
 hpg_store_info.drop(['latitude','longitude'],inplace=True,axis=1)
 store_id_relation = pd.read_csv('.. /input/store_id_relation.csv')
 storeinfo = air_store_info.merge(store_id_relation,on='air_store_id',how='left')
 storeinfo = storeinfo.merge(hpg_store_info,on='hpg_store_id',how='left')
 air_reserve = pd.read_csv('.. /input/air_reserve.csv',parse_dates=['visit_datetime'])
 air_reserve['visit_date'] = air_reserve.visit_datetime.apply(lambda df :
datetime.datetime(year=df.year, month=df.month, day=df.day))
hpg_reserve =pd.read_csv('.. /input/hpg_reserve.csv',parse_dates=['visit_datetime'])
 hpg_reserve['visit_date'] = hpg_reserve.visit_datetime.apply(lambda df :
datetime.datetime(year=df.year, month=df.month, day=df.day))
date_info = pd.read_csv('.. /input/date_info.csv',parse_dates=['calendar_date'] ).rename(columns={'calendar_date':'visit_date'})
 air_reserve_by_date = air_reserve.groupby(['air_store_id','visit_date'] ).reserve_visitors.sum().reset_index(drop=False)
 hpg_reserve_by_date = hpg_reserve.groupby(['hpg_store_id','visit_date'] ).reserve_visitors.sum().reset_index(drop=False)
 test = air_visit_data.merge(storeinfo,on='air_store_id',how='left')
 test = test.merge(air_reserve_by_date,on=['air_store_id','visit_date'],how='left')
 test = test.merge(hpg_reserve_by_date,on=['hpg_store_id','visit_date'],how='left')
 test = test.merge(date_info,on='visit_date',how='left')
 test['year'] = test.visit_date.dt.year
test['month'] = test.visit_date.dt.month
test.reserve_visitors_x = test.reserve_visitors_x.fillna(0)
 test.reserve_visitors_y = test.reserve_visitors_y.fillna(0)
 test.reserve_visitors_x = np.log1p(test.reserve_visitors_x)
 test.reserve_visitors_y = np.log1p(test.reserve_visitors_y)
 test.visitors = np.log1p(test.visitors)
 test = test.fillna(-1)
 test = test.sort_values(by=['visit_date','air_store_id'],ascending=False)
 test = test.reset_index(drop=True)
 return test[list(['id'])+list(columns)]
def test_model_nb(model_num, model_epoch,output_dim=7):
model = NeuralNet(embedding_matrix, 6 ).cuda()
model.load_state_dict(torch.load('.. /input/lstm-numberbatch-{}/model_{}.pth'.format(model_num, model_epoch)) ['model'])
 model.eval()
with torch.no_grad() :
test_preds = np.zeros(( len(x_test_seq), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
return test_preds<find_best_params>
train = MungeTrain()
delta = train.visit_date.max() -pd.Timedelta(weeks=5)
 lastfiveweekstrain = train[train.visit_date>=delta].copy()
lastfiveweekstrain = lastfiveweekstrain.reset_index(drop=True)
 train = train[train.visit_date<delta].copy()
train = train.reset_index(drop=True)
 test = MungeTest(train.columns )
test_model_preds['lstm_nb_0_4'] = test_model_nb(0, 4)[:, 0]
test_model_preds['lstm_nb_1_4'] = test_model_nb(1, 4)[:, 0]
test_model_preds['lstm_nb_2_4'] = test_model_nb(2, 4)[:, 0]<find_best_params>
deltavals = [1,2,3,4,5]
xgbtrainpreds = None
xgbtestpreds = None
for i,deltaweek in enumerate(deltavals):
print(i)
 delta = pd.Timedelta(weeks=deltaweek)
 blindmin = train.visit_date.max() -delta
blindmax = train.visit_date.max()
vismin = blindmin-delta-pd.Timedelta(days=1)
 vismax = blindmax-delta-pd.Timedelta(days=1)
 btrain = None
for x in range(int(60./deltaweek)) :
vistrain = train[(train.visit_date<=vismax)].copy()
blindtrain = train[(train.visit_date>=blindmin)&(train.visit_date<=blindmax)].copy()
features = ['air_genre_name',
'air_area_name', 'hpg_store_id',
'hpg_genre_name', 'hpg_area_name',
'day_of_week', 'holiday_flg']
for c in features:
blindtrain[c+'_Count_Store'] = np.nan
blindtrain[c+'_Mean_Store'] = np.nan
blindtrain[c+'_Median_Store'] = np.nan
blindtrain[c+'_Max_Store'] = np.nan
blindtrain[c+'_Min_Store'] = np.nan
blindtrain[c+'_Std_Store'] = np.nan
blindtrain[[c+'_Count_Store',c+'_Mean_Store',
c+'_Median_Store',c+'_Max_Store',
c+'_Min_Store', c+'_Std_Store']] = LeaveOneOut(vistrain,
blindtrain,
list(['air_store_id']),
c,
useLOO=False,
cut=0 ).values
features = ['air_store_id',
'air_genre_name',
'air_area_name', 'hpg_store_id',
'hpg_genre_name', 'hpg_area_name',
'day_of_week', 'holiday_flg']
for c in features:
blindtrain[c+'_Count'] = np.nan
blindtrain[c+'_Mean'] = np.nan
blindtrain[c+'_Median'] = np.nan
blindtrain[c+'_Max'] = np.nan
blindtrain[c+'_Min'] = np.nan
blindtrain[c+'_Std'] = np.nan
blindtrain[[c+'_Count',c+'_Mean',
c+'_Median',c+'_Max',
c+'_Min', c+'_Std']] = LeaveOneOut(vistrain,
blindtrain,
list([]),
c,
useLOO=False,
cut=0,
addNoise=False ).values
if('air_store_id'!=c):
blindtrain.drop(c,inplace=True,axis=1)
 if(btrain is None):
btrain = blindtrain.copy()
else:
btrain = pd.concat([btrain,blindtrain])
 vismax -= pd.Timedelta(weeks=deltaweek)
 vismin -= pd.Timedelta(weeks=deltaweek)
 blindmin -= pd.Timedelta(weeks=deltaweek)
 blindmax -= pd.Timedelta(weeks=deltaweek)
 vistrain = train.copy()
features = ['air_genre_name',
'air_area_name', 'hpg_store_id',
'hpg_genre_name', 'hpg_area_name',
'day_of_week', 'holiday_flg']
for c in features:
lastfiveweekstrain[c+'_Count_Store'] = np.nan
lastfiveweekstrain[c+'_Mean_Store'] = np.nan
lastfiveweekstrain[c+'_Median_Store'] = np.nan
lastfiveweekstrain[c+'_Max_Store'] = np.nan
lastfiveweekstrain[c+'_Min_Store'] = np.nan
lastfiveweekstrain[c+'_Std_Store'] = np.nan
lastfiveweekstrain[[c+'_Count_Store',c+'_Mean_Store',
c+'_Median_Store',c+'_Max_Store',
c+'_Min_Store', c+'_Std_Store']] = LeaveOneOut(vistrain,
lastfiveweekstrain,
list(['air_store_id']),
c,
useLOO=False,
cut=0 ).values
test[c+'_Count_Store'] = np.nan
test[c+'_Mean_Store'] = np.nan
test[c+'_Median_Store'] = np.nan
test[c+'_Max_Store'] = np.nan
test[c+'_Min_Store'] = np.nan
test[c+'_Std_Store'] = np.nan
test[[c+'_Count_Store',c+'_Mean_Store',
c+'_Median_Store',c+'_Max_Store',
c+'_Min_Store', c+'_Std_Store']] = LeaveOneOut(vistrain,
test,
list(['air_store_id']),
c,
useLOO=False,
cut=0 ).values
features = ['air_store_id',
'air_genre_name',
'air_area_name', 'hpg_store_id',
'hpg_genre_name', 'hpg_area_name',
'day_of_week', 'holiday_flg']
for c in features:
lastfiveweekstrain[c+'_Count'] = np.nan
lastfiveweekstrain[c+'_Mean'] = np.nan
lastfiveweekstrain[c+'_Median'] = np.nan
lastfiveweekstrain[c+'_Max'] = np.nan
lastfiveweekstrain[c+'_Min'] = np.nan
lastfiveweekstrain[c+'_Std'] = np.nan
lastfiveweekstrain[[c+'_Count',c+'_Mean',
c+'_Median',c+'_Max',
c+'_Min', c+'_Std']] = LeaveOneOut(vistrain,
lastfiveweekstrain,
list([]),
c,
useLOO=False,
cut=0,
addNoise=False ).values
test[c+'_Count'] = np.nan
test[c+'_Mean'] = np.nan
test[c+'_Median'] = np.nan
test[c+'_Max'] = np.nan
test[c+'_Min'] = np.nan
test[c+'_Std'] = np.nan
test[[c+'_Count',c+'_Mean',
c+'_Median',c+'_Max',
c+'_Min', c+'_Std']] = LeaveOneOut(vistrain,
test,
list([]),
c,
useLOO=False,
cut=0,
addNoise=False ).values
d_train = xgb.DMatrix(btrain[btrain.columns[3:]], label=btrain.visitors)
 d_valid = xgb.DMatrix(lastfiveweekstrain[btrain.columns[3:]], label=lastfiveweekstrain.visitors)
 d_test = xgb.DMatrix(test[btrain.columns[3:]])
 params = {}
params['objective'] = 'reg:linear'
params['eval_metric'] = 'rmse'
params['eta'] = 0.1
params['max_depth'] = 7
params['subsample']= 0.8
params['colsample_bytree']= 0.8
params['silent'] = 1
watchlist = [(d_train, 'train'),(d_valid, 'valid')]
clf = xgb.train(params, d_train, 1000, watchlist, early_stopping_rounds=20, verbose_eval=10)
 xgbx = clf.predict(d_valid,ntree_limit=clf.best_iteration+1)
 xgby = clf.predict(d_test,ntree_limit=clf.best_iteration+1)
 trainpreds = pd.DataFrame()
trainpreds['visit_date'] = lastfiveweekstrain.visit_date.values
trainpreds['air_store_id'] = lastfiveweekstrain.air_store_id.values
trainpreds['visitors'] = lastfiveweekstrain.visitors.values
trainpreds['weeks_'+str(deltaweek)] = xgbx
testpreds = pd.DataFrame()
testpreds['visit_date'] = test.visit_date.values
testpreds['air_store_id'] = test.air_store_id.values
testpreds['weeks_'+str(deltaweek)] = xgby
if(xgbtrainpreds is None):
xgbtrainpreds = trainpreds.copy()
xgbtestpreds = testpreds.copy()
else:
xgbtrainpreds = xgbtrainpreds.merge(trainpreds[['air_store_id','visit_date','weeks_'+str(deltaweek)]],on=['air_store_id','visit_date'])
 xgbtestpreds = xgbtestpreds.merge(testpreds[['air_store_id','visit_date','weeks_'+str(deltaweek)]],on=['air_store_id','visit_date'] )
twitter_matrix, unknown_words_twitter = build_matrix(tokenizer.word_index, TWITTER_EMBEDDING_PATH, dim=200)
 print('n unknown words(twitter): ', len(unknown_words_twitter))
embedding_matrix = twitter_matrix
print(embedding_matrix.shape)
 del twitter_matrix
gc.collect()<find_best_model_class>
X_train, X_test, y_train, y_test = train_test_split(xgbtrainpreds[['weeks_1', 'weeks_2', 'weeks_3', 'weeks_4', 'weeks_5']],
xgbtrainpreds.visitors, test_size=0.2, random_state=42)
 d_train = xgb.DMatrix(X_train, label=y_train)
 d_valid = xgb.DMatrix(X_test, label=y_test)
 d_test = xgb.DMatrix(xgbtestpreds[['weeks_1', 'weeks_2', 'weeks_3', 'weeks_4', 'weeks_5']])
 params = {}
params['objective'] = 'reg:linear'
params['eval_metric'] = 'rmse'
params['eta'] = 0.2
params['max_depth'] = 3
params['subsample']=0.8
params['colsample_bytree']=0.8
params['silent'] = 1
watchlist = [(d_train, 'train'),(d_valid, 'valid')]
clf = xgb.train(params, d_train, 250, watchlist, early_stopping_rounds=20, verbose_eval=10)
 xgbx = clf.predict(d_valid,ntree_limit=clf.best_iteration+1)
 xgby = clf.predict(d_test,ntree_limit=clf.best_iteration+1 )
def test_model_tw(model_num, model_epoch,output_dim=7):
model = NeuralNet(embedding_matrix, 6 ).cuda()
model.load_state_dict(torch.load('.. /input/lstm-twitter-{}/model_{}.pth'.format(model_num, model_epoch)) ['model'])
 model.eval()
with torch.no_grad() :
test_preds = np.zeros(( len(x_test_seq), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
return test_preds<categorify>
bestrounds = clf.best_iteration+1
d_train = xgb.DMatrix(xgbtrainpreds[['weeks_1', 'weeks_2', 'weeks_3', 'weeks_4', 'weeks_5']], label=xgbtrainpreds.visitors)
 clf = xgb.train(params, d_train, int(( bestrounds)*1.2), verbose_eval=10)
 xgbx = clf.predict(d_train,ntree_limit=bestrounds)
 xgby = clf.predict(d_test,ntree_limit=bestrounds )
test_model_preds['lstm_tw_0_3'] = test_model_tw(0, 3)[:, 0]
test_model_preds['lstm_tw_1_3'] = test_model_tw(1, 3)[:, 0]
test_model_preds['lstm_tw_2_3'] = test_model_tw(2, 3)[:, 0]<define_variables>
print(np.sqrt(mean_squared_error(xgbtrainpreds.visitors.ravel() ,
xgbx)) )
class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
repl = {
"&lt;3": " good ",
":d": " good ",
":dd": " good ",
":p": " good ",
"8)": " good ",
":-)": " good ",
":)": " good ",
";)": " good ",
"(-:": " good ",
"(:": " good ",
"yay!": " good ",
"yay": " good ",
"yaay": " good ",
"yaaay": " good ",
"yaaaay": " good ",
"yaaaaay": " good ",
":/": " bad ",
":&gt;": " sad ",
":')": " sad ",
":-(": " bad ",
":(": " bad ",
":s": " bad ",
":-s": " bad ",
"&lt;3": " heart ",
":d": " smile ",
":p": " smile ",
":dd": " smile ",
"8)": " smile ",
":-)": " smile ",
":)": " smile ",
";)": " smile ",
"(-:": " smile ",
"(:": " smile ",
":/": " worry ",
":&gt;": " angry ",
":')": " sad ",
":-(": " sad ",
":(": " sad ",
":s": " sad ",
":-s": " sad ",
r"\br\b": "are",
r"\bu\b": "you",
r"\bhaha\b": "ha",
r"\bhahaha\b": "ha",
r"\bdon't\b": "do not",
r"\bdoesn't\b": "does not",
r"\bdidn't\b": "did not",
r"\bhasn't\b": "has not",
r"\bhaven't\b": "have not",
r"\bhadn't\b": "had not",
r"\bwon't\b": "will not",
r"\bwouldn't\b": "would not",
r"\bcan't\b": "can not",
r"\bcannot\b": "can not",
r"\bi'm\b": "i am",
"m": "am",
"r": "are",
"u": "you",
"haha": "ha",
"hahaha": "ha",
"don't": "do not",
"doesn't": "does not",
"didn't": "did not",
"hasn't": "has not",
"haven't": "have not",
"hadn't": "had not",
"won't": "will not",
"wouldn't": "would not",
"can't": "can not",
"cannot": "can not",
"i'm": "i am",
"m": "am",
"i'll" : "i will",
"its" : "it is",
"it's" : "it is",
"'s" : " is",
"that's" : "that is",
"weren't" : "were not",
}
re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
 def tokenize(s):
return re_tok.sub(r' \1 ', s ).split()
cont_patterns = [ 
(b'US', b'United States'), 
(b'IT', b'Information Technology'), 
(b'(W|w)on't', b'will not'), 
(b'(C|c)an't', b'can not'), 
(b'(I|i)'m', b'i am'), 
(b'(A|a)in't', b'is not'), 
(b'(\w+)'ll', b'\g<1> will'), 
(b'(\w+)n't', b'\g<1> not'), 
(b'(\w+)'ve', b'\g<1> have'), 
(b'(\w+)'s', b'\g<1> is'), 
(b'(\w+)'re', b'\g<1> are'), 
(b'(\w+)'d', b'\g<1> would'),
]
patterns = [(re.compile(regex), repl)for(regex, repl)in cont_patterns]
def prepare_for_char_n_gram(text):

 in r""
clean = bytes(text.lower() , encoding="utf-8")
 and \t
clean = clean.replace(b"
", b" ")
 clean = clean.replace(b"\t", b" ")
 clean = clean.replace(b"\b", b" ")
 clean = clean.replace(b"\r", b" ")
 for(pattern, repl)in patterns:
clean = re.sub(pattern, repl, clean)
 exclude = re.compile(b'[%s]' % re.escape(bytes(string.punctuation, encoding='utf-8')))
 clean = b" ".join([exclude.sub(b'', token)for token in clean.split() ])
 clean = re.sub(b"\d+", b" ", clean)
 clean = re.sub(b'\s+', b' ', clean)
 clean = re.sub(b'\s+$', b'', clean)
 clean = re.sub(b" ", b"
clean = b"
return str(clean, 'utf-8')
 def count_regexp_occ(regexp="", text=None):

 return len(re.findall(regexp, text))
def get_indicators_and_clean_comments(df):

 ", x))
df["raw_word_len"] = df["comment_text"].apply(lambda x: len(x.split()))
df["raw_char_len"] = df["comment_text"].apply(lambda x: len(x))
df["nb_upper"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[A-Z]", x))
df["nb_fk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[Ff]\S{2}[Kk]", x))
df["nb_sk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[Ss]\S{2}[Kk]", x))
df["nb_dk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[dD]ick", x))
df["nb_you"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\W[Yy]ou\W", x))
df["nb_mother"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\Wmother\W", x))
df["nb_ng"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\Wnigger\W", x))
df["start_with_columns"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"^\:+", x))
df["has_timestamp"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\d{2}|:\d{2}", x))
df["has_date_long"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\D\d{2}:\d{2}, \d{1,2} \w+ \d{4}", x))
df["has_date_short"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\D\d{1,2} \w+ \d{4}", x))
df["has_mail"] = df["comment_text"].apply(
 lambda x: count_regexp_occ(r'[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+', x)
)
 df["has_emphasize_equal"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\={2}.+\={2}", x))
df["has_emphasize_quotes"] = df["comment_text"].apply(lambda x: count_regexp_occ(r""{4}\S+"{4}", x))
df["clean_comment"] = df["comment_text"].apply(lambda x: prepare_for_char_n_gram(x))
df["clean_word_len"] = df["clean_comment"].apply(lambda x: len(x.split()))
df["clean_char_len"] = df["clean_comment"].apply(lambda x: len(x))
df["clean_chars"] = df["clean_comment"].apply(lambda x: len(set(x)))
 df["clean_chars_ratio"] = df["clean_comment"].apply(lambda x: len(set(x)))/ df["clean_comment"].apply(
 lambda x: 1 + min(99, len(x)))
 fts = ["raw_word_len", "raw_char_len", "nb_upper", "nb_fk", "nb_sk", "nb_dk", "nb_you", "nb_mother", "nb_ng", "start_with_columns",
"has_timestamp", "has_date_long", "has_date_short", "has_mail", "has_emphasize_equal", "has_emphasize_quotes", "clean_word_len",
"clean_char_len", "clean_chars", "clean_chars_ratio"]
def preprocess(df):
keys = [i for i in repl.keys() ]
new_data = []
ltr = df["comment_text"].tolist()
for i in tqdm(ltr):
arr = str(i ).split()
xx = ""
for j in arr:
j = str(j ).lower()
if j[:4] == 'http' or j[:3] == 'www':
continue
if j in keys:
j = repl[j]
xx += j + " "
new_data.append(xx)
 df["new_comment_text"] = new_data
trate = df["new_comment_text"].tolist()
for i, c in enumerate(trate):
trate[i] = re.sub('[^a-zA-Z ?!]+', '', str(trate[i] ).lower())
 df["comment_text"] = trate
df.drop(["new_comment_text"], axis=1, inplace=True)
 df_text = df['comment_text']
re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
 get_indicators_and_clean_comments(df)
 return df<categorify>
xgbtestpreds['id'] = xgbtestpreds["air_store_id"]+'_'+xgbtestpreds["visit_date"].map(str)
 xgbtestpreds['id'] = xgbtestpreds.id.str[:-9]
xgbtestpreds['visitors'] = np.expm1(xgby )
<feature_engineering><EOS> 
xgbtestpreds[['id','visitors']].to_csv('xgbtimeseries.csv',index=False )