Kaggle competition: Bike Sharing Demand
<SOS> metric: RMSLE Kaggle data source: bike-sharing-demand<predict_on_test>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import os
import datetime
import calendar
train_good = train[train['res'] == 1].reset_index(drop=True)
 train_good<filter>
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error
train_outlier[train_outlier['diff_num']==0]<filter>
from xgboost import XGBRegressor
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
train_good[train_good['diff_num']==0]<filter>
warnings.filterwarnings("ignore" )
train_outlier[train_outlier['share']==1]<filter>
train_data = pd.read_csv(path_in + 'train.csv', parse_dates = ['datetime'],
index_col='datetime', infer_datetime_format=True)
 test_data = pd.read_csv(path_in + 'test.csv', parse_dates = ['datetime'],
index_col='datetime', infer_datetime_format=True)
 samp_subm = pd.read_csv(path_in+'sampleSubmission.csv', parse_dates = ['datetime'],
index_col='datetime', infer_datetime_format=True )
train_good[train_good['share']==1]<filter>
def rmse(y_true, y_pred):

 return K.sqrt(K.mean(K.square(y_pred - y_true)) )
len(train_outlier[train_outlier["text"].str.find(' ')== -1] )<filter>
num_months_per_year = 12
year_list = [2011, 2012]
len(train_good[train_good["text"].str.find(' ')== -1] )<feature_engineering>
month = 5
year = 2011
start_date = datetime.datetime(year, month, 1, 0, 0, 0)
 end_date = datetime.datetime(year, month, 19, 23, 0, 0)
def subtext_analysis(col, subtext, df1, str1, df2, str2):
result = pd.DataFrame(columns = ['subtext', str1, str2, 'share,%'])
 if(len(df1)> 0)and(len(df2)> 0):
for i in range(len(subtext)) :
result.loc[i,'subtext'] = subtext[i]
num1 = len(df1[df1[col].str.find(subtext[i])> -1])
 result.loc[i, str1] = num1
num2 = len(df2[df2[col].str.find(subtext[i])> -1])
 result.loc[i, str2] = num2
result.loc[i,'share,%'] = round(num1*100/num2,1)if num2 != 0 else 0
print('Number of all data is', len(df2))
display(result.sort_values(by=['share,%', str1], ascending=False))<feature_engineering>
train_data_temp = pd.DataFrame(columns=train_data.columns)
 for year in year_list:
for month in range(num_months_per_year):
start_date = datetime.datetime(year, month+1, 1, 0, 0, 0)
 end_date = datetime.datetime(year, month+1, 19, 23, 0, 0)
 temp = train_data[start_date:end_date].resample('H' ).asfreq()
features_fill_zero = ['casual', 'registered', 'count']
temp[features_fill_zero] = temp[features_fill_zero].fillna(0)
 features_fill_bbfil = ['season', 'holiday', 'workingday', 'weather']
temp[features_fill_bbfil] = temp[features_fill_bbfil].fillna(method='bfill')
 features_fill_linear = ['temp', 'atemp', 'humidity', 'windspeed']
temp[features_fill_linear] = temp[features_fill_linear].interpolate(method='linear')
 train_data_temp = train_data_temp.append(temp)
 train_data = train_data_temp
def subtext_analysis_one_df(col, subtext, df, str):
result = pd.DataFrame(columns = ['subtext', str, 'share of all,%'])
 num_all = len(df)
 if(num_all > 0):
for i in range(len(subtext)) :
result.loc[i,'subtext'] = subtext[i]
num = len(df[df[col].str.find(subtext[i])> -1])
 result.loc[i, str] = num
result.loc[i,'share of all,%'] = round(num*100/num_all,1)
 print('Number of all data is', len(df))
display(result.sort_values(by='share of all,%', ascending=False))<define_variables>
train_data['weekday'] = train_data.index.weekday
train_data['hour'] = train_data.index.hour
test_data['weekday'] = test_data.index.weekday
test_data['hour'] = test_data.index.hour
subtext_test = ['SAD', 'bullying', 'Uh', 'oh', 'onna', 'fun', 'addicted', 'Power', 'well', 'unhappy', 'funny', 'Tears', 'Fears', 'sleeeeepy', ' ', ',', '?', '!' ,'!!', '!!!', ':/', '...', 'http', '****']<compute_test_metric>
def hour_group(s):
if(( 0<=s)&(s<=6)) :
return 1
elif(( s==7)|(s==9)) :
return 2
elif(( s==8)|(s==16)|(s==19)) :
return 3
elif(( 10<=s)&(s<=15)) :
return 4
elif(( s==17)|(s==18)) :
return 5
elif(20<=s):
return 6
subtext_analysis("selected_text", subtext_test, train_outlier, 'train_outliers', train, 'train_all' )<compute_test_metric>
train_data['hour_group'] = train_data['hour'].apply(hour_group)
 test_data['hour_group'] = test_data['hour'].apply(hour_group )
subtext_analysis("selected_text", subtext_test, train_good, 'train_good', train, 'train_all' )<compute_test_metric>
subtext_analysis_one_df("selected_text", subtext_test, train, 'test_all' )<compute_test_metric>
features_one_hot = ['weekday', 'hour_group', 'weather']
train_data[features_one_hot] = train_data[features_one_hot].astype(int ).astype(str)
 test_data[features_one_hot] = test_data[features_one_hot].astype(int ).astype(str )
subtext_analysis_one_df("selected_text", subtext_test, test, 'test_all' )<feature_engineering>
train_data = pd.get_dummies(train_data)
 test_data = pd.get_dummies(test_data )
test['text_chr3'] = test['text'].apply(rep_3chr)
 test.head(10 )<train_model>
scale_features = ['temp', 'atemp', 'humidity', 'hour', 'windspeed']
train_outlier1 = train_outlier.nsmallest(1000, 'metric', keep='all')
 train_outlier2 = train_outlier.nsmallest(2000, 'metric', keep='all')
 train_outlier3 = train_outlier.nsmallest(3000, 'metric', keep='all')
 train_outlier5 = train_outlier.nsmallest(5000, 'metric', keep='all')
 train_outlier8 = train_outlier.nsmallest(8000, 'metric', keep='all' )<compute_test_metric>
scaler = MinMaxScaler()
train_data[scale_features] = scaler.fit_transform(train_data[scale_features])
 test_data[scale_features] = scaler.transform(test_data[scale_features] )
subtext_analysis("selected_text", subtext_test, train_outlier1, 'in worst 1000 outliers', train_outlier, 'in all outliers' )<compute_test_metric>
feature_list = ['holiday', 'workingday', 'weather', 'temp', 'atemp',
'humidity', 'windspeed', 'hour_group',
'hour', 'weekday']
no_features = ['casual', 'registered', 'count']
subtext_analysis("selected_text", subtext_test, train_outlier2, 'in worst 2000 outliers', train_outlier, 'in all outliers' )<compute_test_metric>
predictions = []
for year in year_list:
for month in range(num_months_per_year):
start_date = datetime.datetime(year, month+1, 1, 0, 0, 0)
 end_date = datetime.datetime(year, month+1, 19, 23, 0, 0)
 X_train = train_data[start_date:end_date][train_data.columns.difference(no_features)].copy()
y_train = train_data[start_date:end_date]['count'].copy()
y_train = np.log1p(y_train)
 model_rfr = RandomForestRegressor(n_estimators=500, n_jobs=-1, max_features='auto')
 model_gbr = GradientBoostingRegressor(n_estimators=1000)
 model_rfr.fit(X_train, y_train)
 model_gbr.fit(X_train, y_train)
 start_date = datetime.datetime(year, month+1, 20, 0, 0, 0)
 last_day_of_month = calendar.monthrange(year, month+1)[1]
end_date = datetime.datetime(year, month+1, last_day_of_month, 23, 0, 0)
 X_test = test_data[start_date:end_date][train_data.columns.difference(no_features)].copy()
y_test_rfr = model_rfr.predict(X_test)
 y_test_gbr = model_rfr.predict(X_test)
 y_test = 0.0 * y_test_gbr + 1.0 * y_test_rfr
y_test = np.expm1(y_test)
 predictions.extend(y_test )
subtext_analysis("selected_text", subtext_test, train_outlier3, 'in worst 3000 outliers', train_outlier, 'in all outliers' )<compute_test_metric>
predictions = [0 if i < 0 else i for i in predictions]
subtext_analysis("selected_text", subtext_test, train_outlier5, 'in worst 5000 outliers', train_outlier, 'in all outliers' )<compute_test_metric>
output = pd.DataFrame({'datetime': test_data.index,
'count': predictions})
 output.to_csv('submission.csv', index=False )