Kaggle competition: UI DS Summer School
features['MasVnrArea'] = features['MasVnrArea'].fillna(0 )<groupby>
pd.set_option("display.max_columns", 500)
 pd.set_option("display.max_rows", 200)
 plt.rcParams['figure.figsize'] = [15, 6]
sns.set_style("darkgrid" )
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].apply(lambda x: x.fillna(x.median()))<feature_engineering>
!pip install pandas-profiling
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics2 = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics2.append(i)
 skew_features = features[numerics2].apply(lambda x: stats.skew(x)).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
for i in skew_index:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))<feature_engineering>
online_sales = pd.read_csv('/kaggle/input/uisummerschool/Online_sales.csv', sep=',')
 online_sales.head()
features['HasPool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['Has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['HasGarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['HasBsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['HasFireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['Total_SQR_Footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['Total_Porch_SF'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
 features = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1 )<categorify>
online_sales [['Date', 'Product SKU', 'Quantity', 'Revenue', 'Tax']]
final_features = pd.get_dummies(features ).reset_index(drop=True)
 print(final_features.shape )<split>
test = online_sales [['Date', 'Product SKU', 'Quantity', 'Revenue', 'Tax']]
X = final_features.iloc[:len(y), :]
X_pred = final_features.iloc[len(X):, :]
X_train = X
X_test = X_pred
y_train = y
print('X', X.shape, 'y', y.shape, 'X_pred', X_pred.shape )<compute_train_metric>
condition1= online_sales['Product SKU'] == 'GGOENEBQ079099'
online_sales[(condition1)]
condition2= online_sales['Quantity'] > 2
online_sales[(condition1)&(condition2)]
def rmse_cv(model):
rmse = np.sqrt(-cross_val_score(model, X_train, y_train, scoring="neg_mean_squared_error", cv = 10))
return(rmse )<choose_model_class>
test = online_sales [['Date', 'Product SKU', 'Quantity', 'Revenue', 'Tax', 'Delivery']]
test['Net_Income'] = test['Revenue'] - test['Tax'] - test['Delivery']
test.head()
model_ridge = Ridge(alpha = 10 )<choose_model_class>
kondisi = test['Tax'].isnull()
test.loc[kondisi, ['Tax']] = 1
model_lasso = make_pipeline(RobustScaler() , Lasso(alpha = 0.0004))<compute_train_metric>
test = online_sales.groupby(['Date'])['Quantity'].sum().reset_index()
test.head()
cv_ridge = rmse_cv(model_ridge ).mean()
cv_lasso = rmse_cv(model_lasso ).mean()<prepare_output>
test = online_sales.groupby(['Date', 'Product SKU'])['Quantity'].sum().reset_index()
test.head()
results = pd.DataFrame({
'Model': ['Ridge','Lasso'],
'Score': [cv_ridge,cv_lasso]})
 result_df = results.sort_values(by='Score', ascending=True ).reset_index(drop=True)
 result_df.head()<predict_on_test>
test = online_sales.groupby(['Date'] ).agg({'Quantity': 'sum',
'Revenue': 'sum',
'Tax': 'sum',
'Product SKU': 'count',
'Transaction ID': 'count',
} ).reset_index()
test.head()
model_lasso.fit(X_train, y_train)
 lasso_pred = np.expm1(model_lasso.predict(X_test))
model_ridge.fit(X_train, y_train)
 ridge_pred = np.expm1(model_ridge.predict(X_test)) 
<compute_test_metric>
online_sales.sort_values(by=['Quantity'], ascending = False ).head(15 )
Stack =(lasso_pred + ridge_pred)/ 2<save_to_csv>
test.rename(index=str, columns={"Quantity": "Total Quantity", "Revenue": "Total Revenue"}, inplace = True)
 test.drop(columns=['Product SKU', 'Transaction ID'], inplace = True)
 test.head()
submission = pd.DataFrame({'Id':test_ID, 'SalePrice':Stack})
 submission.to_csv('submission.csv', index=False )<load_from_csv>
online_sales = online_sales = pd.read_csv('/kaggle/input/uisummerschool/Online_sales.csv')
 backup = online_sales.copy()
daily_online_revenue = online_sales.groupby(['Date'])['Revenue'].sum().reset_index()
daily_online_revenue.tail()
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_data = pd.read_csv('/kaggle/input/titanic/test.csv' )<feature_engineering>
add_data = [['2017-12-01', 0], ['2017-12-02', 0], ['2017-12-03', 0],
['2017-12-04', 0], ['2017-12-05', 0], ['2017-12-06', 0],
['2017-12-07', 0], ['2017-12-08', 0], ['2017-12-09', 0],
['2017-12-10', 0], ['2017-12-11', 0], ['2017-12-10', 0],
['2017-12-13', 0], ['2017-12-14', 0]
]
add_data_df = pd.DataFrame(add_data, columns = ['Date', 'Revenue'])
 add_data_df['Date'] = add_data_df['Date'].astype(str)
 add_data_df['Date'] = pd.to_datetime(add_data_df['Date'])
 daily_online_revenue = daily_online_revenue.append(add_data_df)
 df_rev = daily_online_revenue.copy()
daily_online_revenue.tail(20 )
full_data['Fare']=full_data['Fare'].map(lambda x: np.log(x)if x>0 else 0 )<feature_engineering>
daily_online_revenue = preprocess(df_rev ).set_index('Date')
 daily_online_revenue.head()
full_data['Cabin'] = full_data['Cabin'].fillna('Unkown')
 full_data['Cabin'].head()<filter>
def split_train_test(dataset, end_of_training_date):
training_data = daily_online_revenue.loc['2017-01-05':end_of_training_date]
testing_data = daily_online_revenue["2017-12-01":]
return training_data, testing_data
full_data[full_data['Embarked'].isnull() ]<count_values>
def split_label_and_predictor(train_or_test_data):
y_data = train_or_test_data['today_revenue']
x_data = train_or_test_data[['d-1_rev','d-2_rev','d-3_rev','d-4_rev','d-5_rev','d-6_rev','d-7_rev','d-8_rev','d-9_rev','d-10_rev']]
return x_data, y_data
full_data['Embarked'].value_counts()<feature_engineering>
end_of_training_date = "2017-11-30"
training_data, test_data = split_train_test(daily_online_revenue,end_of_training_date)
full_data['Embarked']=full_data['Embarked'].fillna('S' )<filter>
x_train, y_train = split_label_and_predictor(training_data)
 x_test, y_test = split_label_and_predictor(test_data)
full_data[full_data['Fare'].isnull() ]<feature_engineering>
def fit(x_train, y_train):
model = RandomForestRegressor(random_state=1)
 model.fit(x_train, y_train)
 return model
def predict(model, x_test):
y_pred = model.predict(x_test)
 return y_pred
model = fit(x_train, y_train)
full_data['Fare'] = full_data['Fare'].fillna(full_data[(full_data['Pclass']==3)&(full_data['Embarked']=='S')&(full_data['Cabin']=='U')]['Fare'].mean() )<count_values>
df_rev2 = df_rev.copy()
n_iteration = len(x_test)
 result = []
for i in range(n_iteration):
y_pred = predict(model, pd.DataFrame(x_test.iloc[i] ).transpose())
 result.append(y_pred[0])
 df_rev2.loc[df_rev2["Date"]==x_test.index[i],"Revenue"] = y_pred
daily_online_revenue = preprocess(df_rev2 ).set_index('Date')
 _, testing_data = split_train_test(daily_online_revenue,end_of_training_date)
 x_test, _ = split_label_and_predictor(testing_data)
 result
full_data['Title']=full_data['Name'].map(lambda x:x.split(',')[1].split('.')[0].strip())
 full_data['Title'].value_counts()<categorify>
comparison = pd.DataFrame({"Prediction":result,"Actual":y_test})
 comparison.index = y_test.index
error = sqrt(mean_squared_error(comparison["Actual"], comparison["Prediction"]))
print("Error Score(RMSE)= {}".format(round(error,2)))
 historical = pd.DataFrame(y_train ).rename(columns={"Revenue":"Actual"} ).tail(14)
 pd.concat([historical,comparison],sort=True ).plot() ;
TitleDict={}
TitleDict['Mr']='Mr'
TitleDict['Mlle']='Miss'
TitleDict['Miss']='Miss'
TitleDict['Master']='Master'
TitleDict['Jonkheer']='Master'
TitleDict['Mme']='Mrs'
TitleDict['Ms']='Mrs'
TitleDict['Mrs']='Mrs'
TitleDict['Don']='Royalty'
TitleDict['Sir']='Royalty'
TitleDict['the Countess']='Royalty'
TitleDict['Dona']='Royalty'
TitleDict['Lady']='Royalty'
TitleDict['Capt']='Officer'
TitleDict['Col']='Officer'
TitleDict['Major']='Officer'
TitleDict['Dr']='Officer'
TitleDict['Rev']='Officer'
full_data['Title']=full_data['Title'].map(TitleDict)
 full_data['Title'].value_counts()<count_values>
formatted_result = pd.DataFrame(result ).reset_index().rename(columns={"index":"Id",0:"Revenue"})
 display(formatted_result)
 formatted_result.to_csv("result.csv",index=False)