Kaggle competition: UI DS Summer School
def create_model() :
model = Sequential()
model.add(Conv2D(32, 5, activation="relu", input_shape=(28, 28, 1)))
 model.add(Conv2D(32, 5, activation="relu"))
model.add(MaxPool2D(pool_size=(2, 2)))
 model.add(Dropout(0.4))
model.add(Conv2D(64, 3, activation="relu", padding='same'))
model.add(Conv2D(64, 3, activation="relu"))
model.add(Dropout(0.4))
model.add(Conv2D(128, 3, activation="relu", padding='same'))
model.add(Conv2D(128, 3, activation="relu"))
model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(256, activation="relu"))
model.add(Dropout(0.5))
model.add(Dense(64, activation="relu"))
model.add(Dropout(0.2))
model.add(Dense(10, activation="softmax"))
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
 return model<choose_model_class>
pd.set_option("display.max_columns", 500)
 pd.set_option("display.max_rows", 200)
 plt.rcParams['figure.figsize'] = [15, 6]
sns.set_style("darkgrid" )
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=2,
factor=0.4,
min_lr=3e-6)
 early_stops = EarlyStopping(monitor='val_acc', min_delta=0, patience=6, verbose=2, mode='auto' )<choose_model_class>
!pip install pandas-profiling
data_aug = ImageDataGenerator(rotation_range=20, width_shift_range=4, height_shift_range=4, zoom_range=0.1 )<train_model>
online_sales = pd.read_csv('/kaggle/input/uisummerschool/Online_sales.csv', sep=',')
 online_sales.head()
history = model.fit_generator(data_aug.flow(X_train, y_train, batch_size=128), steps_per_epoch=len(X_train)//128,
validation_data=(X_test, y_test), epochs=100, verbose=1, callbacks=[learning_rate_reduction] )<save_to_csv>
online_sales [['Date', 'Product SKU', 'Quantity', 'Revenue', 'Tax']]
def make_submission(model, filename="submission.csv"):
df = pd.read_csv(".. /input/test.csv")
 X = df.values / 255
X = X.reshape(X.shape[0], 28, 28, 1)
 preds = model.predict_classes(X)
 subm = pd.DataFrame(data=list(zip(range(1, len(preds)+ 1), preds)) , columns=["ImageId", "Label"])
 subm.to_csv(filename, index=False)
<save_to_csv>
test = online_sales [['Date', 'Product SKU', 'Quantity', 'Revenue', 'Tax']]
make_submission(model, "submission.csv" )<train_model>
condition1= online_sales['Product SKU'] == 'GGOENEBQ079099'
online_sales[(condition1)]
condition2= online_sales['Quantity'] > 2
online_sales[(condition1)&(condition2)]
print(f"Finished in {int(time.time() - start_time)} seconds..." )<set_options>
test = online_sales [['Date', 'Product SKU', 'Quantity', 'Revenue', 'Tax', 'Delivery']]
test['Net_Income'] = test['Revenue'] - test['Tax'] - test['Delivery']
test.head()
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.width', 1000)
 if not sys.warnoptions:
warnings.simplefilter("ignore" )<define_variables>
kondisi = test['Tax'].isnull()
test.loc[kondisi, ['Tax']] = 1
desc = {
'SalePrice': "The property's sale price in dollars.This is the target variable to be predicted",
'MSSubClass': "The building class",
'MSZoning': "The general zoning classification",
'LotFrontage': "Linear feet of street connected to property",
'LotArea': "Lot size in square feet",
'Street': "Type of road access",
'Alley': "Type of alley access",
'LotShape': "General shape of property",
'LandContour': "Flatness of the property",
'Utilities': "Type of utilities available",
'LotConfig': "Lot configuration",
'LandSlope': "Slope of property",
'Neighborhood': "Physical locations within Ames city limits",
'Condition1': "Proximity to main road or railroad",
'Condition2': "Proximity to main road or railroad(if a second is present)",
'BldgType': "Type of dwelling",
'HouseStyle': "Style of dwelling",
'OverallQual': "Overall material and finish quality",
'OverallCond': "Overall condition rating",
'YearBuilt': "Original construction date",
'YearRemodAdd': "Remodel date",
'RoofStyle': "Type of roof",
'RoofMatl': "Roof material",
'Exterior1st': "Exterior covering on house",
'Exterior2nd': "Exterior covering on house(if more than one material)",
'MasVnrType': "Masonry veneer type",
'MasVnrArea': "Masonry veneer area in square feet",
'ExterQual': "Exterior material quality",
'ExterCond': "Present condition of the material on the exterior",
'Foundation': "Type of foundation",
'BsmtQual': "Height of the basement",
'BsmtCond': "General condition of the basement",
'BsmtExposure': "Walkout or garden level basement walls",
'BsmtFinType1': "Quality of basement finished area",
'BsmtFinSF1': "Type 1 finished square feet",
'BsmtFinType2': "Quality of second finished area(if present)",
'BsmtFinSF2': "Type 2 finished square feet",
'BsmtUnfSF': "Unfinished square feet of basement area",
'TotalBsmtSF': "Total square feet of basement area",
'Heating': "Type of heating",
'HeatingQC': "Heating quality and condition",
'CentralAir': "Central air conditioning",
'Electrical':" Electrical system",
'1stFlrSF': "First Floor square feet",
'2ndFlrSF': "Second floor square feet",
'LowQualFinSF': "Low quality finished square feet(all floors)",
'GrLivArea': "Above grade(ground)living area square feet",
'BsmtFullBath': "Basement full bathrooms",
'BsmtHalfBath': "Basement half bathrooms",
'FullBath': "Full bathrooms above grade",
'HalfBath': "Half baths above grade",
'Bedroom': "Number of bedrooms above basement level",
'Kitchen': "Number of kitchens",
'KitchenQual': "Kitchen quality",
'TotRmsAbvGrd': "Total rooms above grade(does not include bathrooms)",
'Functional': "Home functionality rating",
'Fireplaces': "Number of fireplaces",
'FireplaceQu': "Fireplace quality",
'GarageType': "Garage location",
'GarageYrBlt': "Year garage was built",
'GarageFinish': "Interior finish of the garage",
'GarageCars': "Size of garage in car capacity",
'GarageArea': "Size of garage in square feet",
'GarageQual': "Garage quality",
'GarageCond': "Garage condition",
'PavedDrive': "Paved driveway",
'WoodDeckSF': "Wood deck area in square feet",
'OpenPorchSF': "Open porch area in square feet",
'EnclosedPorch': "Enclosed porch area in square feet",
'3SsnPorch': "Three season porch area in square feet",
'ScreenPorch': "Screen porch area in square feet",
'PoolArea': "Pool area in square feet",
'PoolQC': "Pool quality",
'Fence': "Fence quality",
'MiscFeature': "Miscellaneous feature not covered in other categories",
'MiscVal': "Value of miscellaneous feature",
'MoSold': "Month Sold",
'YrSold': "Year Sold",
'SaleType': "Type of sale",
'SaleCondition': "Condition of sale"
}<load_from_disk>
test = online_sales.groupby(['Date'])['Quantity'].sum().reset_index()
test.head()
json.dump(desc, open("desc.json",'w'))
with open("desc.json", "r")as read_file:
d= json.load(read_file )<load_from_csv>
test = online_sales.groupby(['Date', 'Product SKU'])['Quantity'].sum().reset_index()
test.head()
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<prepare_x_and_y>
test = online_sales.groupby(['Date'] ).agg({'Quantity': 'sum',
'Revenue': 'sum',
'Tax': 'sum',
'Product SKU': 'count',
'Transaction ID': 'count',
} ).reset_index()
test.head()
train_ID = train['Id']
test_ID = test['Id']
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True)
 numerical = [x for x in train.columns if train.dtypes[x] != 'object']
numerical.remove('SalePrice')
 categorical = [x for x in train.columns if train.dtypes[x] == 'object']
y = train.SalePrice.reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 data_overview(features )<prepare_x_and_y>
online_sales.sort_values(by=['Quantity'], ascending = False ).head(15 )
train["SalePrice"] = np.log1p(train["SalePrice"])
 y = train.SalePrice.reset_index(drop=True )<sort_values>
test.rename(index=str, columns={"Quantity": "Total Quantity", "Revenue": "Total Revenue"}, inplace = True)
 test.drop(columns=['Product SKU', 'Transaction ID'], inplace = True)
 test.head()
corr_dict = train.corr() ['SalePrice'][(train.corr() ['SalePrice'] > 0.5)&(train.corr() ['SalePrice'] < 1.0)].sort_values(ascending=False ).to_dict()
print("Higher correlations between predictor variables and the response variable ")
 print("-------------------------------------------------------------------------")
 for x in corr_dict.keys() :
print("(" , round(corr_dict[x],3), ")",x, ":" , " -",desc[x] )<data_type_conversions>
online_sales = online_sales = pd.read_csv('/kaggle/input/uisummerschool/Online_sales.csv')
 backup = online_sales.copy()
daily_online_revenue = online_sales.groupby(['Date'])['Revenue'].sum().reset_index()
daily_online_revenue.tail()
features['MSSubClass'] = features['MSSubClass'].apply(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str )<data_type_conversions>
add_data = [['2017-12-01', 0], ['2017-12-02', 0], ['2017-12-03', 0],
['2017-12-04', 0], ['2017-12-05', 0], ['2017-12-06', 0],
['2017-12-07', 0], ['2017-12-08', 0], ['2017-12-09', 0],
['2017-12-10', 0], ['2017-12-11', 0], ['2017-12-10', 0],
['2017-12-13', 0], ['2017-12-14', 0]
]
add_data_df = pd.DataFrame(add_data, columns = ['Date', 'Revenue'])
 add_data_df['Date'] = add_data_df['Date'].astype(str)
 add_data_df['Date'] = pd.to_datetime(add_data_df['Date'])
 daily_online_revenue = daily_online_revenue.append(add_data_df)
 df_rev = daily_online_revenue.copy()
daily_online_revenue.tail(20 )
for f in('MasVnrType','Alley','PoolQC', 'Fence', 'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',):
features[f] = features[f].fillna('None' )<data_type_conversions>
daily_online_revenue['d-1_rev'] = daily_online_revenue['Revenue'].shift(1)
 daily_online_revenue['d-2_rev'] = daily_online_revenue['Revenue'].shift(2)
 daily_online_revenue['d-3_rev'] = daily_online_revenue['Revenue'].shift(3)
 daily_online_revenue['d-4_rev'] = daily_online_revenue['Revenue'].shift(4)
 daily_online_revenue['d-5_rev'] = daily_online_revenue['Revenue'].shift(5)
 daily_online_revenue = daily_online_revenue.dropna()
features['MiscFeature'] = features['MiscFeature'].fillna('None' )<categorify>
train_data = daily_online_revenue.loc[:'2017-11-30']
test_data = daily_online_revenue.loc['2017-12-1':]
for f in('Exterior1st', 'Exterior2nd', 'SaleType','Utilities'):
features[f] = features[f].fillna(features[f].mode() [0])
 features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))<filter>
x_train = train_data.drop('Revenue', 1)
 df = train_data[['Revenue']].to_string(index=False ).split('
')
 y_train =pd.DataFrame({'Revenue': df})
 y_train=y_train.drop(y_train.index[[0]])
 x_test = test_data.drop('Revenue', 1)
 df = test_data[['Revenue']].to_string(index=False ).split('
')
 y_test =pd.DataFrame({'Revenue': df})
 y_test=y_test.drop(y_test.index[[0]])
 y_test
features[features['GarageYrBlt'] == 2207][['YearBuilt','YearRemodAdd','GarageYrBlt']]<feature_engineering>
def fit(x_train, y_train):
model = RandomForestRegressor(random_state=1)
 model.fit(x_train, y_train)
 return model
def predict(model, x_test):
y_pred = model.predict(x_test)
 return y_pred
model = fit(x_train, y_train)
features.GarageYrBlt.iloc[2592] = 2007<data_type_conversions>
def preprocess(dataset):
processed_dataset = dataset.copy()
processed_dataset['d-1_rev'] = processed_dataset['Revenue'].shift(1)
 processed_dataset['d-2_rev'] = processed_dataset['Revenue'].shift(2)
 processed_dataset['d-3_rev'] = processed_dataset['Revenue'].shift(3)
 processed_dataset['d-4_rev'] = processed_dataset['Revenue'].shift(4)
 processed_dataset['d-5_rev'] = processed_dataset['Revenue'].shift(4)
 processed_dataset = processed_dataset.dropna()
return processed_dataset
def split_label_and_predictor(train_or_test_data):
x_data = train_or_test_data.drop('Revenue', 1)
 df = train_or_test_data[['Revenue']].to_string(index=False ).split('
')
 y_data =pd.DataFrame({'Revenue': df})
 y_data=y_data.drop(y_data.index[[0]])
 return x_data, y_data
def split_train_test(dataset, end_of_training_date):
training_data = dataset.loc[:end_of_training_date]
testing_data = dataset.loc["2017-12-1":]
return training_data, testing_data
df_rev2 = df_rev.copy()
n_iteration = len(x_test)
 result = []
for i in range(n_iteration):
y_pred = predict(model, pd.DataFrame(x_test.iloc[i] ).transpose())
 result.append(y_pred[0])
 df_rev2.loc[df_rev2["Date"]==x_test.index[i],"Revenue"] = y_pred
daily_online_revenue = preprocess(df_rev2 ).set_index('Date')
 _, testing_data = split_train_test(daily_online_revenue,end_of_training_date)
 x_test, _ = split_label_and_predictor(testing_data)
 result
for f in ['GarageYrBlt', 'GarageArea', 'GarageCars']:
features[f] = features[f].fillna(0 )<data_type_conversions>
comparison = pd.DataFrame({"Prediction":result,"Actual":y_test['Revenue']})
 comparison.index = y_test.index
error = sqrt(mean_squared_error(comparison["Actual"], comparison["Prediction"]))
print("Error Score(RMSE)= {}".format(round(error,2)))
 historical = pd.DataFrame(y_train ).rename(columns={"Revenue":"Actual"} ).tail(14)
 pd.concat([historical,comparison],sort=True ).plot() ;
for f in ['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath']:
features[f] = features[f].fillna(0 )<feature_engineering>
formatted_result = pd.DataFrame(result ).reset_index().rename(columns={"index":"Id",0:"Revenue"})
 display(formatted_result)
 formatted_result[['Id', 'Revenue']].to_csv("result.csv",index=False )