Kaggle competition: Dogs vs. Cats Redux: Kernels Edition
<SOS> metric: LogLoss Kaggle data source: dogs-vs.-cats-redux:-kernels-edition<choose_model_class>
!pip install --use-feature=2020-resolver https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds/xgboost-1.3.0_SNAPSHOT%2B00b0ad1293b4fa74d6aca5da4e9ab7a9d16777f0-py3-none-manylinux2010_x86_64.whl
model = Sequential()<choose_model_class>
xgb.__version__
model.add(Conv2D(32,(3,3), strides=(1, 1), padding='same', activation="relu",input_shape =(28,28,1),data_format = "channels_last", use_bias = True))
model.add(Conv2D(32,(3,3), strides=(1, 1), padding='same', activation="relu", use_bias = True))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2), strides=(2,2), padding='same'))
model.add(Dropout(0.2))
model.add(Conv2D(64,(3,3), strides=(1, 1), padding='same', activation="relu", use_bias = True))
model.add(Conv2D(64,(3,3), strides=(1, 1), padding='same', activation="relu", use_bias = True))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2), strides=(2,2), padding='same'))
model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256,activation = "relu", use_bias = True))
model.add(Dropout(0.5))
model.add(Dense(10,activation = "softmax",use_bias = True)) 
<choose_model_class>
import numpy as np
import pandas as pd
import zipfile
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import log_loss, roc_auc_score
import gc
import os
optimizer = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0 )<choose_model_class>
sample_submission = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')
 sample_submission.tail()
model.compile(optimizer = optimizer,loss = "categorical_crossentropy",metrics = ['accuracy'] )<choose_model_class>
TRAIN_FOLDER = '.. /input/cats-and-dogs-embedded-data/train/train/'
TEST_FOLDER = '.. /input/cats-and-dogs-embedded-data/test/test/'
IMG_SIZE = 224
learning_rate_reduction = callbacks.ReduceLROnPlateau(monitor='loss',patience=3, verbose=1,factor=0.2,min_lr=0.00001 )<init_hyperparams>
train_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/train_image_list.npy')
 test_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/test_image_list.npy' )
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(train_x )<train_model>
ids = [int(x.split('.')[0])for x in test_image_list]
ids[:20]
model.fit_generator(datagen.flow(train_x,train_y,batch_size = 100),epochs = 30,steps_per_epoch=train_x.shape[0] // 100, callbacks=[learning_rate_reduction] )<predict_on_test>
target = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/target.npy' )
y_pred = model.predict(test )<prepare_output>
train_NAS_large = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_2/train_NAS_large.npy')
 test_NAS_large = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_2/test_NAS_large.npy' )
y_pred = np.array(y_pred )<define_variables>
X_train, X_val, y_train, y_val = train_test_split(train_NAS_large, target, test_size=0.1, random_state=42 )
y_pred_final = []
for i in y_pred:
y_pred_final.append(np.argmax(i)) 
<save_to_csv>
xgb_params = {
'eta': 0.05,
'max_depth': 4,
'subsample': 0.8,
'colsample_bytree': 0.7,
'objective': 'reg:logistic',
'eval_metric': 'auc',
'verbosity': 0,
'tree_method': 'gpu_hist',
'predictor': 'gpu_predictor'
}
dtrain = xgb.DMatrix(X_train, y_train)
 dval = xgb.DMatrix(X_val, y_val)
 dtest = xgb.DMatrix(test_NAS_large )
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("digit_mnist.csv",index=False )<save_to_csv>
del train_NAS_large, test_NAS_large, X_train, X_val
gc.collect()
def create_download_link(df, title = "Download CSV file", filename = "data.csv"):
csv = df.to_csv()
b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html)
 %matplotlib inline
print(os.listdir(".. /input"))
train_df = pd.read_csv(".. /input/train.csv")
 train2 = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 test2 = pd.read_csv(".. /input/test.csv")
 train_df.shape
def rstr(df, pred=None):
obs = df.shape[0]
types = df.dtypes
counts = df.apply(lambda x: x.count())
 uniques = df.apply(lambda x: [x.unique() ])
 nulls = df.apply(lambda x: x.isnull().sum())
 distincts = df.apply(lambda x: x.unique().shape[0])
 missing_ratio =(df.isnull().sum() / obs)* 100
skewness = df.skew()
kurtosis = df.kurt()
print('Data shape:', df.shape)
 if pred is None:
cols = ['types', 'counts', 'distincts', 'nulls', 'missing ratio', 'uniques', 'skewness', 'kurtosis']
str = pd.concat([types, counts, distincts, nulls, missing_ratio, uniques, skewness, kurtosis], axis = 1)
 else:
corr = df.corr() [pred]
str = pd.concat([types, counts, distincts, nulls, missing_ratio, uniques, skewness, kurtosis, corr], axis = 1, sort=False)
 corr_col = 'corr ' + pred
cols = ['types', 'counts', 'distincts', 'nulls', 'missing_ratio', 'uniques', 'skewness', 'kurtosis', corr_col ]
str.columns = cols
dtypes = str.types.value_counts()
print('___________________________
Data types:
',str.types.value_counts())
 print('___________________________')
 return str
details = rstr(train_df, 'SalePrice')
 display(details.sort_values(by='corr SalePrice', ascending=False))
plt.figure(figsize=(12, 6))
sns.heatmap(train_df.isnull())
 plt.show()<feature_engineering>
%%time
model = xgb.train(xgb_params, dtrain, num_boost_round=110 )
train_df.drop(train_df[train_df["GrLivArea"] > 4000].index, inplace=True)
 train2.drop(train2[train2["GrLivArea"] > 4000].index, inplace=True)
 test_df.loc[666, "GarageQual"] = "TA"
test_df.loc[666, "GarageCond"] = "TA"
test_df.loc[666, "GarageFinish"] = "Unf"
test_df.loc[666, "GarageYrBlt"] = "1980"
test_df.loc[1116, "GarageType"] = np.nan
lot_frontage_by_neighborhood = train_df["LotFrontage"].groupby(train_df["Neighborhood"])
 display(test_df.loc[test_df.GarageYrBlt==2207, ['GarageYrBlt', 'YearBuilt']])
 test_df.loc[test_df.GarageYrBlt==2207.0, 'GarageYrBlt'] = 2007.0
display(test_df.loc[test_df.GarageYrBlt==2207, ['GarageYrBlt', 'YearBuilt']])
<categorify>
val_preds = model.predict(dval )
le = LabelEncoder()
def factorize(df, factor_df, column, fill_na=None):
factor_df[column] = df[column]
if fill_na is not None:
factor_df[column].fillna(fill_na, inplace=True)
 le.fit(factor_df[column].unique())
 factor_df[column] = le.transform(factor_df[column])
 return factor_df
def munge(df):
all_df = pd.DataFrame(index = df.index)
 all_df["LotFrontage"] = df["LotFrontage"]
for key, group in lot_frontage_by_neighborhood:
idx =(df["Neighborhood"] == key)&(df["LotFrontage"].isnull())
 all_df.loc[idx, "LotFrontage"] = group.median()
all_df["LotArea"] = df["LotArea"]
all_df["MasVnrArea"] = df["MasVnrArea"]
all_df["MasVnrArea"].fillna(0, inplace=True)
 all_df["BsmtFinSF1"] = df["BsmtFinSF1"]
all_df["BsmtFinSF1"].fillna(0, inplace=True)
 all_df["BsmtFinSF2"] = df["BsmtFinSF2"]
all_df["BsmtFinSF2"].fillna(0, inplace=True)
 all_df["BsmtUnfSF"] = df["BsmtUnfSF"]
all_df["BsmtUnfSF"].fillna(0, inplace=True)
 all_df["TotalBsmtSF"] = df["TotalBsmtSF"]
all_df["TotalBsmtSF"].fillna(0, inplace=True)
 all_df["1stFlrSF"] = df["1stFlrSF"]
all_df["2ndFlrSF"] = df["2ndFlrSF"]
all_df["GrLivArea"] = df["GrLivArea"]
all_df["GarageArea"] = df["GarageArea"]
all_df["GarageArea"].fillna(0, inplace=True)
 all_df["WoodDeckSF"] = df["WoodDeckSF"]
all_df["OpenPorchSF"] = df["OpenPorchSF"]
all_df["EnclosedPorch"] = df["EnclosedPorch"]
all_df["3SsnPorch"] = df["3SsnPorch"]
all_df["ScreenPorch"] = df["ScreenPorch"]
all_df["BsmtFullBath"] = df["BsmtFullBath"]
all_df["BsmtFullBath"].fillna(0, inplace=True)
 all_df["BsmtHalfBath"] = df["BsmtHalfBath"]
all_df["BsmtHalfBath"].fillna(0, inplace=True)
 all_df["FullBath"] = df["FullBath"]
all_df["HalfBath"] = df["HalfBath"]
all_df["BedroomAbvGr"] = df["BedroomAbvGr"]
all_df["KitchenAbvGr"] = df["KitchenAbvGr"]
all_df["TotRmsAbvGrd"] = df["TotRmsAbvGrd"]
all_df["Fireplaces"] = df["Fireplaces"]
all_df["GarageCars"] = df["GarageCars"]
all_df["GarageCars"].fillna(0, inplace=True)
 all_df["CentralAir"] =(df["CentralAir"] == "Y")* 1.0
all_df["OverallQual"] = df["OverallQual"]
all_df["OverallCond"] = df["OverallCond"]
qual_dict = {None: 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5}
all_df["ExterQual"] = df["ExterQual"].map(qual_dict ).astype(int)
 all_df["ExterCond"] = df["ExterCond"].map(qual_dict ).astype(int)
 all_df["BsmtQual"] = df["BsmtQual"].map(qual_dict ).astype(int)
 all_df["BsmtCond"] = df["BsmtCond"].map(qual_dict ).astype(int)
 all_df["HeatingQC"] = df["HeatingQC"].map(qual_dict ).astype(int)
 all_df["KitchenQual"] = df["KitchenQual"].map(qual_dict ).astype(int)
 all_df["FireplaceQu"] = df["FireplaceQu"].map(qual_dict ).astype(int)
 all_df["GarageQual"] = df["GarageQual"].map(qual_dict ).astype(int)
 all_df["GarageCond"] = df["GarageCond"].map(qual_dict ).astype(int)
 all_df["BsmtExposure"] = df["BsmtExposure"].map(
 {None: 0, "No": 1, "Mn": 2, "Av": 3, "Gd": 4} ).astype(int)
 bsmt_fin_dict = {None: 0, "Unf": 1, "LwQ": 2, "Rec": 3, "BLQ": 4, "ALQ": 5, "GLQ": 6}
all_df["BsmtFinType1"] = df["BsmtFinType1"].map(bsmt_fin_dict ).astype(int)
 all_df["BsmtFinType2"] = df["BsmtFinType2"].map(bsmt_fin_dict ).astype(int)
 all_df["Functional"] = df["Functional"].map(
 {None: 0, "Sal": 1, "Sev": 2, "Maj2": 3, "Maj1": 4,
"Mod": 5, "Min2": 6, "Min1": 7, "Typ": 8} ).astype(int)
 all_df["GarageFinish"] = df["GarageFinish"].map(
 {None: 0, "Unf": 1, "RFn": 2, "Fin": 3} ).astype(int)
 all_df["Fence"] = df["Fence"].map(
 {None: 0, "MnWw": 1, "GdWo": 2, "MnPrv": 3, "GdPrv": 4} ).astype(int)
 all_df["YearBuilt"] = df["YearBuilt"]
all_df["YearRemodAdd"] = df["YearRemodAdd"]
all_df["GarageYrBlt"] = df["GarageYrBlt"]
all_df["GarageYrBlt"].fillna(0.0, inplace=True)
 all_df["MoSold"] = df["MoSold"]
all_df["YrSold"] = df["YrSold"]
all_df["LowQualFinSF"] = df["LowQualFinSF"]
all_df["MiscVal"] = df["MiscVal"]
all_df["PoolQC"] = df["PoolQC"].map(qual_dict ).astype(int)
 all_df["PoolArea"] = df["PoolArea"]
all_df["PoolArea"].fillna(0, inplace=True)
 all_df = factorize(df, all_df, "MSSubClass")
 all_df = factorize(df, all_df, "MSZoning", "RL")
 all_df = factorize(df, all_df, "LotConfig")
 all_df = factorize(df, all_df, "Neighborhood")
 all_df = factorize(df, all_df, "Condition1")
 all_df = factorize(df, all_df, "BldgType")
 all_df = factorize(df, all_df, "HouseStyle")
 all_df = factorize(df, all_df, "RoofStyle")
 all_df = factorize(df, all_df, "Exterior1st", "Other")
 all_df = factorize(df, all_df, "Exterior2nd", "Other")
 all_df = factorize(df, all_df, "MasVnrType", "None")
 all_df = factorize(df, all_df, "Foundation")
 all_df = factorize(df, all_df, "SaleType", "Oth")
 all_df = factorize(df, all_df, "SaleCondition")
 all_df["IsRegularLotShape"] =(df["LotShape"] == "Reg")* 1
all_df["IsLandLevel"] =(df["LandContour"] == "Lvl")* 1
all_df["IsLandSlopeGentle"] =(df["LandSlope"] == "Gtl")* 1
all_df["IsElectricalSBrkr"] =(df["Electrical"] == "SBrkr")* 1
all_df["IsGarageDetached"] =(df["GarageType"] == "Detchd")* 1
all_df["IsPavedDrive"] =(df["PavedDrive"] == "Y")* 1
all_df["HasShed"] =(df["MiscFeature"] == "Shed")* 1.
 all_df["Remodeled"] =(all_df["YearRemodAdd"] != all_df["YearBuilt"])* 1
all_df["RecentRemodel"] =(all_df["YearRemodAdd"] == all_df["YrSold"])* 1
all_df["VeryNewHouse"] =(all_df["YearBuilt"] == all_df["YrSold"])* 1
all_df["Has2ndFloor"] =(all_df["2ndFlrSF"] == 0)* 1
all_df["HasMasVnr"] =(all_df["MasVnrArea"] == 0)* 1
all_df["HasWoodDeck"] =(all_df["WoodDeckSF"] == 0)* 1
all_df["HasOpenPorch"] =(all_df["OpenPorchSF"] == 0)* 1
all_df["HasEnclosedPorch"] =(all_df["EnclosedPorch"] == 0)* 1
all_df["Has3SsnPorch"] =(all_df["3SsnPorch"] == 0)* 1
all_df["HasScreenPorch"] =(all_df["ScreenPorch"] == 0)* 1
all_df["HighSeason"] = df["MoSold"].replace(
 {1: 0, 2: 0, 3: 0, 4: 1, 5: 1, 6: 1, 7: 1, 8: 0, 9: 0, 10: 0, 11: 0, 12: 0})
 all_df["NewerDwelling"] = df["MSSubClass"].replace(
 {20: 1, 30: 0, 40: 0, 45: 0,50: 0, 60: 1, 70: 0, 75: 0, 80: 0, 85: 0,
90: 0, 120: 1, 150: 0, 160: 0, 180: 0, 190: 0})
 all_df.loc[df.Neighborhood == 'NridgHt', "Neighborhood_Good"] = 1
all_df.loc[df.Neighborhood == 'Crawfor', "Neighborhood_Good"] = 1
all_df.loc[df.Neighborhood == 'StoneBr', "Neighborhood_Good"] = 1
all_df.loc[df.Neighborhood == 'Somerst', "Neighborhood_Good"] = 1
all_df.loc[df.Neighborhood == 'NoRidge', "Neighborhood_Good"] = 1
all_df["Neighborhood_Good"].fillna(0, inplace=True)
 all_df["SaleCondition_PriceDown"] = df.SaleCondition.replace(
 {'Abnorml': 1, 'Alloca': 1, 'AdjLand': 1, 'Family': 1, 'Normal': 0, 'Partial': 0})
 all_df["BoughtOffPlan"] = df.SaleCondition.replace(
 {"Abnorml" : 0, "Alloca" : 0, "AdjLand" : 0, "Family" : 0, "Normal" : 0, "Partial" : 1})
 all_df["BadHeating"] = df.HeatingQC.replace(
 {'Ex': 0, 'Gd': 0, 'TA': 0, 'Fa': 1, 'Po': 1})
 area_cols = ['LotFrontage', 'LotArea', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF',
'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'GrLivArea', 'GarageArea', 'WoodDeckSF',
'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'LowQualFinSF', 'PoolArea' ]
all_df["TotalArea"] = all_df[area_cols].sum(axis=1)
 all_df["TotalArea1st2nd"] = all_df["1stFlrSF"] + all_df["2ndFlrSF"]
all_df["Age"] = 2010 - all_df["YearBuilt"]
all_df["TimeSinceSold"] = 2010 - all_df["YrSold"]
all_df["SeasonSold"] = all_df["MoSold"].map({12:0, 1:0, 2:0, 3:1, 4:1, 5:1,
6:2, 7:2, 8:2, 9:3, 10:3, 11:3} ).astype(int)
 all_df["YearsSinceRemodel"] = all_df["YrSold"] - all_df["YearRemodAdd"]
all_df["SimplOverallQual"] = all_df.OverallQual.replace(
 {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2, 6 : 2, 7 : 3, 8 : 3, 9 : 3, 10 : 3})
 all_df["SimplOverallCond"] = all_df.OverallCond.replace(
 {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2, 6 : 2, 7 : 3, 8 : 3, 9 : 3, 10 : 3})
 all_df["SimplPoolQC"] = all_df.PoolQC.replace(
 {1 : 1, 2 : 1, 3 : 2, 4 : 2})
 all_df["SimplGarageCond"] = all_df.GarageCond.replace(
 {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})
 all_df["SimplGarageQual"] = all_df.GarageQual.replace(
 {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})
 all_df["SimplFireplaceQu"] = all_df.FireplaceQu.replace(
 {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})
 all_df["SimplFunctional"] = all_df.Functional.replace(
 {1 : 1, 2 : 1, 3 : 2, 4 : 2, 5 : 3, 6 : 3, 7 : 3, 8 : 4})
 all_df["SimplKitchenQual"] = all_df.KitchenQual.replace(
 {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})
 all_df["SimplHeatingQC"] = all_df.HeatingQC.replace(
 {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})
 all_df["SimplBsmtFinType1"] = all_df.BsmtFinType1.replace(
 {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2, 6 : 2})
 all_df["SimplBsmtFinType2"] = all_df.BsmtFinType2.replace(
 {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2, 6 : 2})
 all_df["SimplBsmtCond"] = all_df.BsmtCond.replace(
 {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})
 all_df["SimplBsmtQual"] = all_df.BsmtQual.replace(
 {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})
 all_df["SimplExterCond"] = all_df.ExterCond.replace(
 {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})
 all_df["SimplExterQual"] = all_df.ExterQual.replace(
 {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})
 neighborhood_map = {
"MeadowV" : 0,
"IDOTRR" : 1,
"BrDale" : 1,
"OldTown" : 1,
"Edwards" : 1,
"BrkSide" : 1,
"Sawyer" : 1,
"Blueste" : 1,
"SWISU" : 2,
"NAmes" : 2,
"NPkVill" : 2,
"Mitchel" : 2,
"SawyerW" : 2,
"Gilbert" : 2,
"NWAmes" : 2,
"Blmngtn" : 2,
"CollgCr" : 2,
"ClearCr" : 3,
"Crawfor" : 3,
"Veenker" : 3,
"Somerst" : 3,
"Timber" : 3,
"StoneBr" : 4,
"NoRidge" : 4,
"NridgHt" : 4,
}
all_df["NeighborhoodBin"] = df["Neighborhood"].map(neighborhood_map)
 return all_df
train_df_munged = munge(train_df)
 test_df_munged = munge(test_df)
 print(train_df_munged.shape)
 print(test_df_munged.shape)
 neighborhood_bin_train = pd.DataFrame(index = train_df.index)
 neighborhood_bin_train["NeighborhoodBin"] = train_df_munged["NeighborhoodBin"]
neighborhood_bin_test = pd.DataFrame(index = test_df.index)
 neighborhood_bin_test["NeighborhoodBin"] = test_df_munged["NeighborhoodBin"]<feature_engineering>
roc_auc_score(y_val, val_preds )
numeric_features = train_df_munged.dtypes[train_df_munged.dtypes != "object"].index
skewed = train_df_munged[numeric_features].apply(lambda x: skew(x.dropna().astype(float)))
 skewed = skewed[skewed > 0.75]
skewedNParray= skewed
skewed = skewed.index
train_df_munged[skewed] = np.log1p(train_df_munged[skewed])
 test_df_munged[skewed] = np.log1p(test_df_munged[skewed])
 df_skew = pd.DataFrame(index=skewed, columns=['Skew', 'Skew after boxcox1p'])
 df_skew['Skew'] = skewedNParray.values
df_skew['Skew after boxcox1p'] = np.log1p(skewedNParray.values)
 fig = plt.figure(figsize=(53, 14))
sns.pointplot(x=df_skew.index, y='Skew', data=df_skew, markers=['o'], linestyles=['-'])
 sns.pointplot(x=df_skew.index, y='Skew after boxcox1p', data=df_skew, markers=['x'], linestyles=['--'], color='
plt.xlabel('Skewed Features', size=15, labelpad=12.5)
 plt.ylabel('Skewness', size=20, labelpad=12.5)
 plt.tick_params(axis='x', labelsize=11)
 plt.tick_params(axis='y', labelsize=15)
 plt.title('Skewed Features Before and After ', size=20)
 plt.show()
plt.show()
scaler = StandardScaler()
scaler.fit(train_df_munged[numeric_features])
 scaled = scaler.transform(train_df_munged[numeric_features])
 for i, col in enumerate(numeric_features):
train_df_munged[col] = scaled[:, i]
scaled = scaler.transform(test_df_munged[numeric_features])
 for i, col in enumerate(numeric_features):
test_df_munged[col] = scaled[:, i] 
<categorify>
log_loss(y_val, val_preds )
def onehot(onehot_df, df, column_name, fill_na, drop_name):
onehot_df[column_name] = df[column_name]
if fill_na is not None:
onehot_df[column_name].fillna(fill_na, inplace=True)
 dummies = pd.get_dummies(onehot_df[column_name], prefix="_" + column_name)
 onehot_df = onehot_df.join(dummies)
 onehot_df = onehot_df.drop([column_name], axis=1)
 return onehot_df
def munge_onehot(df):
onehot_df = pd.DataFrame(index = df.index)
 onehot_df = onehot(onehot_df, df, "MSSubClass", None, "40")
 onehot_df = onehot(onehot_df, df, "MSZoning", "RL", "RH")
 onehot_df = onehot(onehot_df, df, "LotConfig", None, "FR3")
 onehot_df = onehot(onehot_df, df, "Neighborhood", None, "OldTown")
 onehot_df = onehot(onehot_df, df, "Condition1", None, "RRNe")
 onehot_df = onehot(onehot_df, df, "BldgType", None, "2fmCon")
 onehot_df = onehot(onehot_df, df, "HouseStyle", None, "1.5Unf")
 onehot_df = onehot(onehot_df, df, "RoofStyle", None, "Shed")
 onehot_df = onehot(onehot_df, df, "Exterior1st", "VinylSd", "CBlock")
 onehot_df = onehot(onehot_df, df, "Exterior2nd", "VinylSd", "CBlock")
 onehot_df = onehot(onehot_df, df, "Foundation", None, "Wood")
 onehot_df = onehot(onehot_df, df, "SaleType", "WD", "Oth")
 onehot_df = onehot(onehot_df, df, "SaleCondition", "Normal", "AdjLand")
 temp_df = df[["MasVnrType", "MasVnrArea"]].copy()
idx =(df["MasVnrArea"] != 0)&(( df["MasVnrType"] == "None")|(df["MasVnrType"].isnull()))
temp_df.loc[idx, "MasVnrType"] = "BrkFace"
onehot_df = onehot(onehot_df, temp_df, "MasVnrType", "None", "BrkCmn")
 onehot_df = onehot(onehot_df, df, "LotShape", None, "IR3")
 onehot_df = onehot(onehot_df, df, "LandContour", None, "Low")
 onehot_df = onehot(onehot_df, df, "LandSlope", None, "Sev")
 onehot_df = onehot(onehot_df, df, "Electrical", "SBrkr", "FuseP")
 onehot_df = onehot(onehot_df, df, "GarageType", "None", "CarPort")
 onehot_df = onehot(onehot_df, df, "PavedDrive", None, "P")
 onehot_df = onehot(onehot_df, df, "MiscFeature", "None", "Othr")
 onehot_df = onehot(onehot_df, df, "Street", None, "Grvl")
 onehot_df = onehot(onehot_df, df, "Alley", "None", "Grvl")
 onehot_df = onehot(onehot_df, df, "Condition2", None, "PosA")
 onehot_df = onehot(onehot_df, df, "RoofMatl", None, "WdShake")
 onehot_df = onehot(onehot_df, df, "Heating", None, "Wall")
 onehot_df = onehot(onehot_df, df, "ExterQual", "None", "Ex")
 onehot_df = onehot(onehot_df, df, "ExterCond", "None", "Ex")
 onehot_df = onehot(onehot_df, df, "BsmtQual", "None", "Ex")
 onehot_df = onehot(onehot_df, df, "BsmtCond", "None", "Ex")
 onehot_df = onehot(onehot_df, df, "HeatingQC", "None", "Ex")
 onehot_df = onehot(onehot_df, df, "KitchenQual", "TA", "Ex")
 onehot_df = onehot(onehot_df, df, "FireplaceQu", "None", "Ex")
 onehot_df = onehot(onehot_df, df, "GarageQual", "None", "Ex")
 onehot_df = onehot(onehot_df, df, "GarageCond", "None", "Ex")
 onehot_df = onehot(onehot_df, df, "PoolQC", "None", "Ex")
 onehot_df = onehot(onehot_df, df, "BsmtExposure", "None", "Gd")
 onehot_df = onehot(onehot_df, df, "BsmtFinType1", "None", "GLQ")
 onehot_df = onehot(onehot_df, df, "BsmtFinType2", "None", "GLQ")
 onehot_df = onehot(onehot_df, df, "Functional", "Typ", "Typ")
 onehot_df = onehot(onehot_df, df, "GarageFinish", "None", "Fin")
 onehot_df = onehot(onehot_df, df, "Fence", "None", "MnPrv")
 onehot_df = onehot(onehot_df, df, "MoSold", None, None)
 year_map = pd.concat(pd.Series("YearBin" + str(i+1), index=range(1871+i*20,1891+i*20)) for i in range(0, 7))
yearbin_df = pd.DataFrame(index = df.index)
 yearbin_df["GarageYrBltBin"] = df.GarageYrBlt.map(year_map)
 yearbin_df["GarageYrBltBin"].fillna("NoGarage", inplace=True)
 yearbin_df["YearBuiltBin"] = df.YearBuilt.map(year_map)
 yearbin_df["YearRemodAddBin"] = df.YearRemodAdd.map(year_map)
 onehot_df = onehot(onehot_df, yearbin_df, "GarageYrBltBin", None, None)
 onehot_df = onehot(onehot_df, yearbin_df, "YearBuiltBin", None, None)
 onehot_df = onehot(onehot_df, yearbin_df, "YearRemodAddBin", None, None)
 return onehot_df
onehot_df = munge_onehot(train_df)
 onehot_df = onehot(onehot_df, neighborhood_bin_train, "NeighborhoodBin", None, None)
 train_df_munged = train_df_munged.join(onehot_df )<define_variables>
%%time
val_shap_preds = model.predict(dval, pred_contribs=True )
drop_cols = [
"_Exterior1st_ImStucc", "_Exterior1st_Stone",
"_Exterior2nd_Other","_HouseStyle_2.5Fin",
"_RoofMatl_Membran", "_RoofMatl_Metal", "_RoofMatl_Roll",
"_Condition2_RRAe", "_Condition2_RRAn", "_Condition2_RRNn",
"_Heating_Floor", "_Heating_OthW",
"_Electrical_Mix",
"_MiscFeature_TenC",
"_GarageQual_Ex", "_PoolQC_Fa"
]
train_df_munged.drop(drop_cols, axis=1, inplace=True)
 onehot_df = munge_onehot(test_df)
 onehot_df = onehot(onehot_df, neighborhood_bin_test, "NeighborhoodBin", None, None)
 test_df_munged = test_df_munged.join(onehot_df)
 test_df_munged.drop(["_MSSubClass_150"], axis=1, inplace=True)
 drop_cols = [
"_Condition2_PosN",
"_MSZoning_C(all)",
"_MSSubClass_160",
]
train_df_munged.drop(drop_cols, axis=1, inplace=True)
 test_df_munged.drop(drop_cols, axis=1, inplace=True)
<prepare_x_and_y>
%%time
test_shap_preds = model.predict(dtest, pred_contribs=True)
label_df = pd.DataFrame(index = train_df_munged.index, columns=["SalePrice"])
 label_df["SalePrice"] = train_df["SalePrice"]
print(list(train_df_munged))
print("Training set size:", train_df_munged.shape)
 print("Test set size:", test_df_munged.shape )<merge>
%%time
train_shap_preds = model.predict(dtrain, pred_contribs=True )
train2=munge(train2)
 colsLinReg =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish',
'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold', 'MSZoning', 'LotConfig', 'Neighborhood',
'Condition1', 'BldgType', 'HouseStyle', 'RoofStyle', 'Exterior1st',
'Exterior2nd', 'MasVnrType', 'MasVnrArea', 'Foundation',
'SaleType', 'SaleCondition')
 for c in colsLinReg:
lbl = LabelEncoder()
lbl.fit(list(train2[c].values))
train2[c] = lbl.transform(list(train2[c].values))
X = train2.values
y = label_df.values
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=7)
 def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
model = linear_model.LinearRegression()
model.fit(X_train, y_train)
 print("Predict value " + str(model.predict([X_test[142]])))
 print("Real value " + str(y_test[142]))
print("Accuracy --> ", model.score(X_test, y_test)*100)
 foo=model.predict(X_test)
 print("RMSE --> ", rmse(y_test, foo)) 
<train_model>
test_preds = model.predict(dtest )
<split><EOS> 
sample_submission['id'] = ids
sample_submission['label'] = test_preds
sample_submission.to_csv('submission.csv', index=False)
 sample_submission.head()