Kaggle competition: Avito Demand Prediction Challenge
<SOS> metric: RMSE Kaggle data source: avito-demand-prediction-challenge<feature_engineering>
color = sns.color_palette()
%matplotlib inline
py.init_notebook_mode(connected=True)
 pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 999
<define_variables>
train_df = pd.read_csv(".. /input/train.csv", parse_dates=["activation_date"])
 test_df = pd.read_csv(".. /input/test.csv", parse_dates=["activation_date"])
 print("Train file rows and columns are : ", train_df.shape)
 print("Test file rows and columns are : ", test_df.shape )
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
n_lag = [3,7,14]
new_df = generate_lag_features(market_train_df,n_lag=n_lag)
<merge>
( train_df.image.loc[0] )
market_train_df = pd.merge(market_train_df,new_df,how='left',on=['time','assetCode'])
 market_train_df.head() 
<categorify>
temp_data = StringIO()
region_df = pd.read_csv(temp_data)
 train_df = pd.merge(train_df, region_df, how="left", on="region" )
def mis_impute(data):
for i in data.columns:
if data[i].dtype == "object":
data[i] = data[i].fillna("other")
 elif(data[i].dtype == "int64" or data[i].dtype == "float64"):
data[i] = data[i].fillna(data[i].mean())
 else:
pass
return data
market_train_df = mis_impute(market_train_df )<categorify>
temp_series = train_df['region_en'].value_counts() 
(temp_series[:5] )
def data_prep(market_train):
market_train['bartrend'] = market_train['close'] / market_train['open']
market_train['average'] =(market_train['close'] + market_train['open'])/2
lbl = {k: v for v, k in enumerate(market_train['assetCode'].unique())}
market_train['assetCodeT'] = market_train['assetCode'].map(lbl)
 market_train = market_train.dropna(axis=0)
 return market_train
market_train_df = data_prep(market_train_df)
 print(market_train_df.shape,market_train_df.columns )<define_variables>
sizes =(np.array(( temp_series / temp_series.sum())*100))
up = market_train_df['returnsOpenNextMktres10'] >= 0
universe = market_train_df['universe'].values
d = market_train_df['time']
fcol = [c for c in market_train_df if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', 'universe','sourceTimestamp']]
X = market_train_df[fcol].values
up = up.values
r = market_train_df.returnsOpenNextMktres10.values
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng)
 assert X.shape[0] == up.shape[0] == r.shape[0]
te = market_train_df['time']>date(2015, 1, 1)
 tt = 0
for tt,i in enumerate(te.values):
if i:
idx = tt
print(i,tt)
 break
print(idx)
 X_train, X_test = X[:idx],X[idx:]
up_train, up_test = up[:idx],up[idx:]
r_train, r_test = r[:idx],r[idx:]
u_train,u_test = universe[:idx],universe[idx:]
d_train,d_test = d[:idx],d[idx:]
train_data = lgb.Dataset(X_train, label=up_train.astype(int))
test_data = lgb.Dataset(X_test, label=up_test.astype(int))<compute_train_metric>
cnt_srs = train_df['city'].value_counts().head(20)
 type(cnt_srs )
confidence_test =(gbm_1.predict(X_test)+ gbm_2.predict(X_test)) /2
confidence_test =(confidence_test-confidence_test.min())/(confidence_test.max() -confidence_test.min())
 confidence_test = confidence_test*2-1
print(max(confidence_test),min(confidence_test))
r_test = r_test.clip(-1,1)
 x_t_i = confidence_test * r_test * u_test
data = {'day' : d_test, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_test = mean / std
print(score_test)
<set_options>
temp_data = StringIO()
temp_df = pd.read_csv(temp_data)
 train_df = pd.merge(train_df, temp_df, on="parent_category_name", how="left" )
del X_train,X_test
gc.collect()<normalization>
temp_data = StringIO()
temp_df = pd.read_csv(temp_data)
 train_df = pd.merge(train_df, temp_df, on="category_name", how="left" )
def post_scaling(df):
mean, std = np.mean(df), np.std(df)
 df =(df - mean)/(std * 8)
 return np.clip(df,-1,1 )<feature_engineering>
cnt_srs = train_df['activation_date']
days = env.get_prediction_days()
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
total_market_obs_df = []
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
if(n_days%50==0):
print(n_days,end=' ')
 t = time.time()
market_obs_df['time'] = market_obs_df['time'].dt.date
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
total_market_obs_df.append(market_obs_df)
 if len(total_market_obs_df)==1:
history_df = total_market_obs_df[0]
else:
history_df = pd.concat(total_market_obs_df[-(np.max(n_lag)+1):])
 print(history_df)
 new_df = generate_lag_features(history_df,n_lag=[3,7,14])
 market_obs_df = pd.merge(market_obs_df,new_df,how='left',on=['time','assetCode'])
 market_obs_df = mis_impute(market_obs_df)
 market_obs_df = data_prep(market_obs_df)
 X_live = market_obs_df[fcol].values
X_live = 1 -(( maxs - X_live)/ rng)
 prep_time += time.time() - t
t = time.time()
lp =(gbm_1.predict(X_live)+ gbm_2.predict(X_live)) /2
prediction_time += time.time() -t
t = time.time()
confidence = lp
confidence =(confidence-confidence.min())/(confidence.max() -confidence.min())
 confidence = confidence * 2 - 1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence': post_scaling(confidence)})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
env.write_submission_file()
sub = pd.read_csv("submission.csv" )<import_modules>
train_df["title_nwords"] = train_df["title"].apply(lambda x: len(x.split()))
test_df["title_nwords"] = test_df["title"].apply(lambda x: len(x.split()))
import numpy as np
import lightgbm as lgb
import pandas as pd
from kaggle.competitions import twosigmanews
import matplotlib.pyplot as plt
import random
from datetime import datetime, date
from xgboost import XGBClassifier
from sklearn import model_selection
from sklearn.metrics import mean_squared_error
import time<import_modules>
tfidf_vec = TfidfVectorizer(ngram_range=(1,1))
full_tfidf = tfidf_vec.fit_transform(train_df['title'].values.tolist() + test_df['title'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['title'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['title'].values.tolist())
env = twosigmanews.make_env()
print('Done!' )<feature_engineering>
n_comp = 3
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
( market_train_df, news_train_df)= env.get_training_data() 
<feature_engineering>
svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
market_train_df['time'] = market_train_df['time'].dt.date
market_train_df = market_train_df.loc[market_train_df['time']>=date(2010, 1, 1)]<filter>
svd_obj.transform(train_tfidf )
market_train_df = market_train_df[market_train_df.assetCode != 'TW.N']
market_train_df = market_train_df[market_train_df.assetCode != 'PGN.N']
market_train_df = market_train_df[market_train_df.assetCode != 'EBRYY.OB']<merge>
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
n_lag = [3,7,14]
new_df = generate_lag_features(market_train_df,n_lag=n_lag)
 market_train_df = pd.merge(market_train_df,new_df,how='left',on=['time','assetCode'] )<categorify>
train_svd.columns = ['svd_title_'+str(i+1)for i in range(n_comp)]
test_svd.columns = ['svd_title_'+str(i+1)for i in range(n_comp)]
def mis_impute(data):
for i in data.columns:
if data[i].dtype == "object":
data[i] = data[i].fillna("other")
 elif(data[i].dtype == "int64" or data[i].dtype == "float64"):
data[i] = data[i].fillna(data[i].mean())
 else:
pass
return data
market_train_df = mis_impute(market_train_df )<categorify>
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd
def data_prep(market_train):
lbl = {k: v for v, k in enumerate(market_train['assetCode'].unique())}
market_train['assetCodeT'] = market_train['assetCode'].map(lbl)
 market_train = market_train.dropna(axis=0)
 return market_train
market_train_df = data_prep(market_train_df)
 print(market_train_df.shape )<define_variables>
test_df["description"].fillna("NA", inplace=True )
up = market_train_df['returnsOpenNextMktres10'] >= 0
universe = market_train_df['universe'].values
d = market_train_df['time']
fcol = [c for c in market_train_df if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', 'date', 'universe','sourceTimestamp']]
X = market_train_df[fcol].values
up = up.values
r = market_train_df.returnsOpenNextMktres10.values
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng)
 assert X.shape[0] == up.shape[0] == r.shape[0]
X_train, X_test, up_train, up_test, r_train, r_test,u_train,u_test,d_train,d_test = model_selection.train_test_split(X, up, r,universe,d, test_size=0.25, random_state=99)
 train_data = lgb.Dataset(X, label=up.astype(int))
test_data = lgb.Dataset(X_test, label=up_test.astype(int))<compute_train_metric>
train_df["desc_nwords"] = train_df["description"].apply(lambda x: len(x.split()))
train_df["desc_nwords"]
confidence_test =(gbm_1.predict(X_test)+ gbm_2.predict(X_test)) /2
confidence_test =(confidence_test-confidence_test.min())/(confidence_test.max() -confidence_test.min())
 confidence_test = confidence_test*2-1
print(max(confidence_test),min(confidence_test))
r_test = r_test.clip(-1,1)
 x_t_i = confidence_test * r_test * u_test
data = {'day' : d_test, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_test = mean / std
print(score_test)
<set_options>
test_df["desc_nwords"] = test_df["description"].apply(lambda x: len(x.split()))
cnt_srs = train_df['desc_nwords'].value_counts()
cnt_srs
del X_train,X_test
gc.collect()<feature_engineering>
tfidf_vec = TfidfVectorizer(ngram_range=(1,1), max_features=100000)
 full_tfidf = tfidf_vec.fit_transform(train_df['description'].values.tolist() + test_df['description'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['description'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['description'].values.tolist())
 n_comp = 3
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_desc_'+str(i+1)for i in range(n_comp)]
test_svd.columns = ['svd_desc_'+str(i+1)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd
days = env.get_prediction_days()
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
total_market_obs_df = []
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
if(n_days%50==0):
print(n_days,end=' ')
 t = time.time()
market_obs_df['time'] = market_obs_df['time'].dt.date
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
total_market_obs_df.append(market_obs_df)
 if len(total_market_obs_df)==1:
history_df = total_market_obs_df[0]
else:
history_df = pd.concat(total_market_obs_df[-(np.max(n_lag)+1):])
 print(history_df)
 new_df = generate_lag_features(history_df,n_lag=[3,7,14])
 market_obs_df = pd.merge(market_obs_df,new_df,how='left',on=['time','assetCode'])
 market_obs_df = mis_impute(market_obs_df)
 market_obs_df = data_prep(market_obs_df)
 X_live = market_obs_df[fcol].values
X_live = 1 -(( maxs - X_live)/ rng)
 prep_time += time.time() - t
t = time.time()
lp =(gbm_1.predict(X_live)+ gbm_2.predict(X_live)) /2
prediction_time += time.time() -t
t = time.time()
confidence = lp
confidence =(confidence-confidence.min())/(confidence.max() -confidence.min())
 confidence = confidence * 2 - 1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':confidence})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
env.write_submission_file()
sub = pd.read_csv("submission.csv" )<import_modules>
train_y = train_df["deal_probability"].values
test_id = test_df["item_id"].values
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from keras.models import Sequential
from keras.layers import Dense, Dropout, LSTM, GRU
from keras.layers.embeddings import Embedding
import lightgbm as lgb
from kaggle.competitions import twosigmanews
from tqdm import tqdm
import gc<define_variables>
train_df["activation_weekday"] = train_df["activation_date"].dt.weekday
test_df["activation_weekday"] = test_df["activation_date"].dt.weekday
DEBUG = False
YEARMIN = 2011<define_variables>
np.unique(train_df[col].values ).shape
np.random.seed(100)
 set_random_seed(150 )<split>
cat_vars = ["region", "city", "parent_category_name", "category_name", "user_type", "param_1", "param_2", "param_3"]
for col in cat_vars:
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))
 train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))
 test_df[col] = lbl.transform(list(test_df[col].values.astype('str')) )
env = twosigmanews.make_env() 
(market_train, news_train)= env.get_training_data()<filter>
cat_vars = ["region", "city", "parent_category_name", "category_name", "user_type", "param_1", "param_2", "param_3"]
for col in cat_vars:
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))
 train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))
 test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))
 cols_to_drop = ["item_id", "user_id", "title", "description", "activation_date", "image"]
train_X = train_df.drop(cols_to_drop + ["region_en", "parent_category_name_en", "category_name_en", "price_new", "deal_probability"], axis=1)
 test_X = test_df.drop(cols_to_drop, axis=1 )
market_train = market_train.loc[market_train['time'].dt.year > YEARMIN]<feature_engineering>
model.best_iteration
<count_missing_values>
def run_lgb(train_X, train_y, val_X, val_y,alpha):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 30,
"learning_rate" : alpha,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.7,
"bagging_frequency" : 5,
"bagging_seed" : 2018,
"verbosity" : -1
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 evals_result = {}
model = lgb.train(params, lgtrain, 1000, valid_sets=[lgval], early_stopping_rounds=10000, verbose_eval=20, evals_result=evals_result)
 return model, evals_result
<set_options><EOS> 
start_time = time.time()
dev_X = train_X.iloc[:-200000,:]
val_X = train_X.iloc[-200000:,:]
dev_y = train_y[:-200000]
val_y = train_y[-200000:]
print(dev_X.shape, val_X.shape, test_X.shape)
 learningList = [0.001,0.003,0.009,0.027,0.081,0.243,0.729,2.178,6.514,19.51]
train_error = list()
test_error = list()
for i in learningList:
print("the learning_rate 0f ",i)
 model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, i)
 pred_train = model.predict(dev_X, num_iteration=model.best_iteration)
 pred_val= model.predict(val_X,num_iteration = model.best_iteration)
 train_error.append(metrics.mean_squared_error(dev_y,pred_train))
test_error.append(metrics.mean_squared_error(val_y,pred_val))
del model, evals_result
plt.plot(learningList,train_error)
 plt.plot(learningList,test_error)
 plt.show()
print("--- %s seconds ---" %(time.time() - start_time))