Kaggle competition: Web Traffic Time Series Forecasting
cols_to_use = data_dict['WNCAATourneySeeds'].columns.difference(train.columns ).tolist() + ['Season']
train = train.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=["Season","TeamID"]),
how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'])
 train.drop(['TeamID'], axis=1, inplace=True)
 train = train.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=["Season","TeamID"]),
how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], suffixes=('_W', '_L'))
train.drop(['TeamID'], axis=1, inplace=True)
 print(train.shape)
 train.head()<merge>
from fbprophet import Prophet
cols_to_use = gameCities.columns.difference(test.columns ).tolist() + ["Season", "WTeamID", "LTeamID"]
test = test.merge(gameCities[cols_to_use].drop_duplicates(subset=["Season", "WTeamID", "LTeamID"]),
how="left", on=["Season", "WTeamID", "LTeamID"])
 del gameCities
gc.collect()
test.head()
cols_to_use = data_dict["WSeasons"].columns.difference(test.columns ).tolist() + ["Season"]
test = test.merge(data_dict["WSeasons"][cols_to_use].drop_duplicates(subset=["Season"]),
how="left", on=["Season"])
 test.head()
cols_to_use = data_dict["WTeams"].columns.difference(test.columns ).tolist()
test = test.merge(data_dict["WTeams"][cols_to_use].drop_duplicates(subset=["TeamID"]),
how="left", left_on=["WTeamID"], right_on=["TeamID"])
 test.drop(['TeamID'], axis=1, inplace=True)
 test = test.merge(data_dict["WTeams"][cols_to_use].drop_duplicates(subset=["TeamID"]),
how="left", left_on=["LTeamID"], right_on=["TeamID"], suffixes=('_W', '_L'))
test.drop(['TeamID'], axis=1, inplace=True)
 test.head()
cols_to_use = data_dict['WNCAATourneySeeds'].columns.difference(test.columns ).tolist() + ['Season']
test = test.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=["Season","TeamID"]),
how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'])
 test.drop(['TeamID'], axis=1, inplace=True)
 test = test.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=["Season","TeamID"]),
how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], suffixes=('_W', '_L'))
test.drop(['TeamID'], axis=1, inplace=True)
 print(test.shape)
 test.head()<drop_column>
train = pd.read_csv("/kaggle/input/web-traffic-time-series-forecasting/train_1.csv.zip" )
not_exist_in_test = [c for c in train.columns.values.tolist() if c not in test.columns.values.tolist() ]
print(not_exist_in_test)
 train = train.drop(not_exist_in_test, axis=1)
 train.head()<groupby>
all_pages = train['Page']
team_win_score = regularSeason.groupby(['Season', 'WTeamID'] ).agg({'WScore':['sum', 'count', 'var']} ).reset_index()
team_win_score.columns = [' '.join(col ).strip() for col in team_win_score.columns.values]
team_loss_score = regularSeason.groupby(['Season', 'LTeamID'] ).agg({'LScore':['sum', 'count', 'var']} ).reset_index()
team_loss_score.columns = [' '.join(col ).strip() for col in team_loss_score.columns.values]
del regularSeason
gc.collect()<merge>
first_page = all_pages[0]
first_page
train = pd.merge(train, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])
 train = pd.merge(train, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])
 train = pd.merge(train, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])
 train = pd.merge(train, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])
 train.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)
 train.head()<merge>
df = pd.DataFrame(train_allT, columns = ['Date',first_page])
 df = df.rename(columns={'Date':'ds', first_page:'y'} )
test = pd.merge(test, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])
 test = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])
 test = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])
 test = pd.merge(test, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])
 test.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)
 test.head()<feature_engineering>
m = Prophet()
m.fit(df )
def preprocess(df):
df['x_score'] = df['WScore sum_x'] + df['LScore sum_y']
df['y_score'] = df['WScore sum_y'] + df['LScore sum_x']
df['x_count'] = df['WScore count_x'] + df['LScore count_y']
df['y_count'] = df['WScore count_y'] + df['WScore count_x']
df['x_var'] = df['WScore var_x'] + df['LScore count_y']
df['y_var'] = df['WScore var_y'] + df['WScore var_x']
return df
train = preprocess(train)
 test = preprocess(test )<drop_column>
future = m.make_future_dataframe(periods=60)
 future.head()
train_win = train.copy()
train_los = train.copy()
train_win = train_win[['Seed_W', 'Seed_L', 'TeamName_W', 'TeamName_L',
'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]
train_los = train_los[['Seed_L', 'Seed_W', 'TeamName_L', 'TeamName_W',
'y_score', 'x_score', 'x_count', 'y_count', 'x_var', 'y_var']]
train_win.columns = ['Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',
'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']
train_los.columns = ['Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',
'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']
test = test[['ID', 'Seed_W', 'Seed_L', 'TeamName_W', 'TeamName_L',
'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]
test.columns = ['ID', 'Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',
'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']<feature_engineering>
forecast = m.predict(future)
 forecast[['ds', 'yhat', 'yhat_lower', 'yhat_upper']].tail()
def feature_engineering(df):
df['Seed_diff'] = df['Seed_1'] - df['Seed_2']
df['Score_diff'] = df['Score_1'] - df['Score_2']
df['Count_diff'] = df['Count_1'] - df['Count_2']
df['Var_diff'] = df['Var_1'] - df['Var_2']
df['Mean_score1'] = df['Score_1'] / df['Count_1']
df['Mean_score2'] = df['Score_2'] / df['Count_2']
df['Mean_score_diff'] = df['Mean_score1'] - df['Mean_score2']
df['FanoFactor_1'] = df['Var_1'] / df['Mean_score1']
df['FanoFactor_2'] = df['Var_2'] / df['Mean_score2']
return df
train_win = feature_engineering(train_win)
 train_los = feature_engineering(train_los)
 test = feature_engineering(test )<concatenate>
result = forecast['yhat']
data = pd.concat(( train_win, train_los)).reset_index(drop=True)
 print(data.shape)
 data.head()<categorify>
sub_result = result[-60:]
categoricals = ["TeamName_1", "TeamName_2"]
for c in categoricals:
le = LabelEncoder()
data[c] = data[c].fillna("NaN")
 data[c] = le.fit_transform(data[c])
 test[c] = le.transform(test[c])
 data.head()<drop_column>
key = pd.read_csv("/kaggle/input/web-traffic-time-series-forecasting/key_1.csv.zip" )
target = 'result'
features = data.columns.values.tolist()
features.remove(target )<train_model>
listOfIds = key.index[key['Page'].str.contains(first_page)].values
lgbm = LgbModel(data, test, target, features, categoricals=categoricals, n_splits=10,
cv_method="StratifiedKFold", group=None, task="classification", scaler=None, verbose=True )<load_from_csv>
sub = pd.read_csv("/kaggle/input/web-traffic-time-series-forecasting/sample_submission_1.csv.zip" )
submission_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')
 submission_df['Pred'] = lgbm.y_pred
submission_df<save_to_csv>
sub['Visits'].loc[listOfIds.min() :listOfIds.max() ] = sub_result.values.round()
submission_df.to_csv('submission.csv', index=False )<load_from_csv>
def my_function() :
for page in all_pages:
df = pd.DataFrame(train_allT, columns = ['Date',page])
 df = df.rename(columns={'Date':'ds', page:'y'})
 m = Prophet()
m.fit(df)
 future = m.make_future_dataframe(periods=60)
 forecast = m.predict(future)
 result = forecast['yhat']
sub_result = result[-60:]
listOfIds = key.index[key['Page'].str.contains(page)].values
sub['Visits'].loc[listOfIds.min() :listOfIds.max() ] = sub_result.values.round()
%matplotlib inline
warnings.filterwarnings('ignore')
 test_data = pd.read_csv('.. /input/restaurant-revenue-prediction/test.csv.zip')
 raw_df = pd.read_csv('.. /input/restaurant-revenue-prediction/train.csv.zip' )<count_missing_values>
print('Reading data...')
 key_1 = pd.read_csv('/kaggle/input/web-traffic-time-series-forecasting/key_2.csv.zip')
 train_1 = pd.read_csv('/kaggle/input/web-traffic-time-series-forecasting/train_2.csv.zip')
 ss_1 = pd.read_csv('/kaggle/input/web-traffic-time-series-forecasting/sample_submission_2.csv.zip')
raw_df.isnull().sum().any()<drop_column>
print('Preprocessing...')
 print('Processing...')
 ids = key_1.Id.values
pages = key_1.Page.values
raw_df.drop('Id',axis=1,inplace=True )<filter>
print('key_1...')
 d_pages = {}
for id, page in zip(ids, pages):
d_pages[id] = page[:-11]
raw_df = raw_df[raw_df['revenue']<8e+06].copy()<feature_engineering>
for i, ss_id in enumerate(ss_ids):
ss_visits[i] = d_visits[d_pages[ss_id]]
print('Saving submission...')
 subm = pd.DataFrame({'Id': ss_ids, 'Visits': ss_visits})
 subm.to_csv('submission.csv', index=False )
df = pd.concat([raw_df,test_data],axis=0)
 df['Open Date'] = pd.to_datetime(df['Open Date'])
 df['launch_Month'] = [x.month for x in df['Open Date']]
df['launch_year'] = [x.year for x in df['Open Date']]
df.drop(['Id','Open Date'],axis=1,inplace=True )<count_values>
warnings.filterwarnings('ignore' )
df['Type'].value_counts()<count_values>
print('Pre-processing and feature engineering train data...')
 train_flattened = pd.melt(train[list(train.columns[-49:])+['Page']], id_vars='Page', var_name='date', value_name='Visits')
 train_flattened['date'] = train_flattened['date'].astype('datetime64[ns]')
 train_flattened['weekend'] =(( train_flattened.date.dt.dayofweek)// 5 == 1 ).astype(float)
df['City Group'].value_counts()<count_values>
times_series_means = pd.DataFrame(mean_group ).reset_index(drop=False )
df['City'].value_counts()<drop_column>
df_date_index = times_series_means[['date','Visits']].set_index('date' )
df.drop('City',axis=1,inplace=True )<categorify>
result = adfuller(df_date_index)
 print(result[0])
 print(result[1] )
df['Type'] = df['Type'].map({'FC':0,'IL':1,'DT':2,'MB':3})
 encoder = OrdinalEncoder()
df['City Group'] = encoder.fit_transform(np.array(df['City Group'] ).reshape(-1,1))
df['City Group'] = df['City Group'].apply(int )<groupby>
order_aic_bic=[]
for p in range(3):
for q in range(3):
try:
model = SARIMAX(df_date_index, order=(p,0,q), seasonal_order=(1,2,0,7))
results = model.fit()
order_aic_bic.append(( p,q, results.aic, results.bic))
print(p,q,results.aic, results.bic)
 except:
print(p, q, None, None )
df.groupby('launch_year')['revenue'].mean()<categorify>
!pip install pmdarima
df.launch_year = df.launch_year.astype(str)
 df.launch_Month = df.launch_Month.astype(str)
 year_dummy = pd.get_dummies(df[['launch_year','launch_Month']],drop_first=True)
 df = pd.concat([df,year_dummy],axis=1)
 df.drop(['launch_year','launch_Month'],axis=1, inplace=True )<drop_column>
import pmdarima as pm
processed_df = df.dropna(axis=0)
 processed_test_data = df[128:].drop('revenue',axis=1)
<split>
model = SARIMAX(df_date_index,
order=(2,0,1),
seasonal_order=(1,1,1,7),
trend='c')
 results = model.fit()
X=processed_df.drop('revenue',axis=1)
 y=df['revenue'][:128]
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.20,random_state=12345 )<create_dataframe>
forecast_object = results.get_forecast(steps=90)
 mean = forecast_object.predicted_mean
conf_int = forecast_object.conf_int()
dates = mean.index
regressors = {
'Linear Regression' : LinearRegression() ,
'Logistic Regression' : LogisticRegression() ,
'Decision Tree' : DecisionTreeRegressor() ,
'Random Forest' : RandomForestRegressor() ,
'Support Vector Machines' : SVR() ,
'K-nearest Neighbors' : KNeighborsRegressor() ,
'XGBoost' : XGBRegressor()
}
results=pd.DataFrame(columns=['MAE','MSE','R2-score'])
 for method,func in regressors.items() :
func.fit(X_train,y_train)
 pred = func.predict(X_test)
 results.loc[method]= [mean_absolute_error(y_test,pred),
mean_squared_error(y_test,pred),
r2_score(y_test,pred)
 ]
results<choose_model_class>
df_date_index.index = pd.to_datetime(df_date_index.index )
parameters = {'learning_rate': [.03, 0.05,.07,.09],
'max_depth': [6,7,8,9],
'min_child_weight': [4],
'subsample': [0.7],
'colsample_bytree': [0.7],
'n_estimators': [500,700]}
xgb_grid = GridSearchCV(XGBRegressor() ,
parameters,
cv = 3,
n_jobs = 5,
verbose=True)
 xgb_grid.fit(X,y)
 print(xgb_grid.best_score_)
 print(xgb_grid.best_params_ )<save_to_csv>
print(mean.iloc[-1])
 print(conf_int.iloc[-1] )
<save_to_csv><EOS> 
y_forecasted = pred.predicted_mean
y_truth = df_date_index['2016-10-01':]
mse =(( y_forecasted - y_truth)** 2 ).mean()
print('The Mean Squared Error of our forecasts is {}'.format(round(mse, 2)))