Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_pretrained>
warnings.filterwarnings('ignore' )
resnet34_classify = load('/kaggle/input/severstalmodels/resnet34_classify.pth' ).cuda()
unet_mobilenet2 = load('/kaggle/input/severstalmodels/unet_mobilenet2.pth' ).cuda()
ures34 = load('/kaggle/input/severstalmodels/unet_resnet34.pth' ).cuda()
urese = load('/kaggle/input/severstalmodels/unet_se_resnext50_32x4d.pth' ).cuda()
device = torch.device("cuda")
 seresnext50 = SCSEUnet(seg_classes = 4, backbone_arch='seresnext50')
 seresnext50 = nn.DataParallel(seresnext50)
 state = torch.load(".. /input/steelfaultmodels0/SEResNext50_combo.pth", map_location=lambda storage, loc: storage)
 seresnext50.load_state_dict(state["state_dict"])
 seresnext50.eval()
seresnext50.to(device)
<categorify>
df_train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv',index_col='Id')
 df_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv', index_col ='Id' )
def create_transforms(additional):
res = list(additional)
 res.extend([
A.Normalize(
 mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)
),
ChannelTranspose()
])
 res = A.Compose(res)
 return res
img_folder = '/kaggle/input/severstal-steel-defect-detection/test_images'
batch_size = 3
num_workers = 0
transforms = [
[],
[A.HorizontalFlip(p=1)],
[A.VerticalFlip(p=1)]
]
transforms = [create_transforms(t)for t in transforms]
datasets = [TtaWrap(ImageDataset(img_folder=img_folder, transforms=t), tfms=t)for t in transforms]
loaders = [DataLoader(d, num_workers=num_workers, batch_size=batch_size, shuffle=False)for d in datasets]<define_search_space>
def load_data() :
data_dir = Path('.. /input/house-prices-advanced-regression-techniques/')
 df_train = pd.read_csv(data_dir / 'train.csv', index_col = 'Id')
 df_test = pd.read_csv(data_dir / 'test.csv', index_col = 'Id')
 df = pd.concat([df_train,df_test])
 df = clean(df)
 df = encode(df)
 df = impute(df)
 df_train = df.loc[df_train.index,:]
df_test = df.loc[df_test.index, :]
return df_train, df_test
thresholds = [0.5, 0.5, 0.5, 0.5]
min_area = [600, 600, 1000, 2000]
thresh_lab = [0.3,0.3,0.3,0.3]
res = []
total = len(datasets[0])//batch_size
for loaders_batch in tqdm_notebook(zip(*loaders), total=total):
preds = []
classes = []
image_file = []
for i, batch in enumerate(loaders_batch):
features = batch['features'].cuda()
p = torch.sigmoid(model(features))
cls = torch.sigmoid(resnet34_classify(features))
p = datasets[i].inverse(p)
 preds.append(p)
 classes.append(cls)
 image_file = batch['image_file']
preds = torch.stack(preds)
 preds = torch.mean(preds, dim=0)
 preds = preds.detach().cpu().numpy()
classes = torch.stack(classes)
 classes = torch.mean(classes, dim=0)
 classes = classes.detach().cpu().numpy()
for p, cls, file in zip(preds, classes, image_file):
file = os.path.basename(file)
 for i in range(4):
p_channel = p[i]
cls_channel = cls[i]
imageid_classid = file+'_'+str(i+1)
 p_channel =(p_channel>thresholds[i] ).astype(np.uint8)
 if p_channel.sum() < min_area[i] or cls_channel < thresh_lab[i]:
p_channel = np.zeros(p_channel.shape, dtype=p_channel.dtype)
 res.append({
'ImageId_ClassId': imageid_classid,
'EncodedPixels': mask2rle(p_channel)
 })
 df = pd.DataFrame(res)
 df.to_csv('submission.csv', index=False )<save_to_csv>
def clean(df):
df['Exterior2nd'] = df['Exterior2nd'].replace({'Brk Cmn': 'BrkComm'})
 df['GarageYrBlt'] = df['GarageYrBlt'].where(df.GarageYrBlt <=2010, df.YearBuilt)
 df.rename(columns = {'1stFlrSF': 'FirstFlrSF', '2ndFlrSF': 'SecondFlrSF', '3SsnPorch':'Threeseasonporch'}, inplace=True)
 return df
df = pd.DataFrame(res)
 df = df.fillna('')
 df.to_csv('submission.csv', index=False )<feature_engineering>
features_nom = ['MSSubClass', 'MSZoning', 'Street', 'Alley','LandContour', 'LotConfig','Neighborhood','Condition1','Condition2','BldgType','HouseStyle','RoofStyle','RoofMatl','Exterior1st','Exterior2nd','MasVnrType','Foundation','Heating','CentralAir','GarageType','MiscFeature','SaleType','SaleCondition']
five_levels = ['Po', 'Fa', 'TA','Gd','Ex']
ten_levels =list(range(10))
ordered_levels = {
'OverallQual':ten_levels,
'OverallCond':ten_levels,
'ExterQual':five_levels,
'ExterCond':five_levels,
'BsmtQual':five_levels,
'BsmtCond':five_levels,
'HeatingQC':five_levels,
'KitchenQual':five_levels,
'FireplaceQu':five_levels,
'GarageQual': five_levels,
'GarageCond':five_levels,
'PoolQC':five_levels,
'LotShape':['Reg', 'IR1','IR2','IR3'],
'LandSlope':['Sev', 'Mod','Gtl'],
'BsmtExposure': ['No','Mn','Av','Gd'],
'BsmtFinType1':['Unf','LwQ','Rec','BLQ','ALQ','GLQ'],
'BsmtFinType2':['Unf','LwQ','Rec','BLQ','ALQ','GLQ'],
'Functional':['Sal','Sev','Maj1','Maj2','Mod','Min2','Min1','Typ'],
'GarageFinish': ['Unf','RFn','Fin'],
'PavedDrive':['N','P','Y'],
'Utilities': ['NoSeWa','NoSewr','AllPub'],
'CentralAir':['N','Y'],
'Electrical':['Mix','FuseP','FuseF','FuseA','SBrkr'],
'Fence':['MnWw','GdWo','MnPrv','GdPrv'],
}
ordered_levels = {key:['None']+value for key, value in ordered_levels.items() }
def encode(df):
for name in features_nom:
df[name] = df[name].astype('category')
 if 'None' not in df[name].cat.categories:
df[name].cat.add_categories('None', inplace=True)
 for name, levels in ordered_levels.items() :
df[name] = df[name].astype(CategoricalDtype(levels, ordered=True))
return df
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 df['empty'] = df['EncodedPixels'].map(lambda x: not x)
 df[df['empty'] == False]['Class'].value_counts()<feature_engineering>
def impute(df):
for name in df.select_dtypes('number'):
df[name] = df[name].fillna(0)
 for name in df.select_dtypes('category'):
df[name] = df[name].fillna('None')
 return df
%matplotlib inline
df = pd.read_csv('submission.csv')[:100]
df['Image'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[0])
 df['Class'] = df['ImageId_ClassId'].map(lambda x: x.split('_')[1])
 for row in df.itertuples() :
img_path = os.path.join(img_folder, row.Image)
 img = cv2.imread(img_path)
 mask = rle2mask(row.EncodedPixels,(1600, 256)) \
if isinstance(row.EncodedPixels, str)else np.zeros(( 256, 1600))
if mask.sum() == 0:
continue
fig, axes = plt.subplots(1, 2, figsize=(20, 60))
axes[0].imshow(img/255)
 axes[1].imshow(mask*60)
 axes[0].set_title(row.Image)
 axes[1].set_title(row.Class)
 plt.show()<import_modules>
def score_dataset(X,y,model = XGBRegressor()):
for colname in X.select_dtypes(['category']):
X[colname] = X[colname].cat.codes
log_y = np.log(y)
 score = cross_val_score(model, X, log_y, cv=5, scoring = 'neg_mean_squared_error')
 score = -1 * score.mean()
score = np.sqrt(score)
 return score
import os
import json
import gc
import cv2
import keras
from keras.models import load_model
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import sys<save_to_csv>
X = df_train.copy()
y = X.pop('SalePrice')
 baseline_score = score_dataset(X,y)
 print(baseline_score )
df = pd.read_csv('.. /input/severstal-steel-defect-detection/sample_submission.csv')
 df.set_index('ImageId_ClassId', inplace=True)
 df_submit = pd.read_csv(".. /input/ece285-sub/unet_res.csv")
 df_submit.set_index('ImageId_ClassId', inplace=True)
 9
for name, row in df_submit.iterrows() :
df.loc[name] = row
df.reset_index(inplace=True)
 df.to_csv('submission.csv', index=False )<import_modules>
def make_mi_scores(X, y):
X = X.copy()
for colname in X.select_dtypes(['object','category']):
X[colname], _ = X[colname].factorize()
discrete_features = [pd.api.types.is_integer_dtype(t)for t in X.dtypes]
mi_scores = mutual_info_regression(X, y, discrete_features = discrete_features, random_state=0)
 mi_scores = pd.Series(mi_scores, name = 'MI Scores', index = X.columns)
 mi_scores = mi_scores.sort_values(ascending = False)
 return mi_scores
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeClassifier, plot_tree
from sklearn.metrics import confusion_matrix 
<load_from_csv>
X = df_train.copy()
y = X.pop('SalePrice')
 mi_scores = make_mi_scores(X,y)
 mi_scores
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 train_data.head()<load_from_csv>
def drop_uninformative(df, mi_scores):
return df.loc[:, mi_scores>0.0]
test_data = pd.read_csv('/kaggle/input/titanic/test.csv')
 test_data.head()<load_from_csv>
X = df_train.copy()
y = X.pop('SalePrice')
 X = drop_uninformative(X, mi_scores)
 score_dataset(X,y )
submission = pd.read_csv('.. /input/titanic/gender_submission.csv' )<set_options>
def label_encode(df):
X = df.copy()
for colname in X.select_dtypes(['category']):
X[colname] = X[colname].cat.codes
return X
def highlight(value):
if value >= 0.75:
style = 'background-color: green'
elif value >= 0.5:
style = 'background-color: palegreen'
elif value >= 0.25:
style = 'background-color: pink'
else:
style = 'background-color: red'
return style<groupby>
def mathematical_transforms(df):
X = pd.DataFrame()
X['LivLotRatio'] = df.GrLivArea / df.LotArea
X['Spaciousness'] =(df.FirstFlrSF + df.SecondFlrSF)/ df.TotRmsAbvGrd
X['Feet'] = np.sqrt(df.GrLivArea)
 X['TotalSF'] = df.TotalBsmtSF + df.FirstFlrSF + df.SecondFlrSF
X['TotalBathrooms'] = df.FullBath + 0.5* df.HalfBath + df.BsmtFullBath + 0.5 * df.BsmtHalfBath
X['TotalPorchSF']= df.OpenPorchSF + df.Threeseasonporch + df.EnclosedPorch + df.ScreenPorch + df.WoodDeckSF
return X
def interactions(df):
X = pd.get_dummies(df.BldgType, prefix = 'Bldg')
 X = X.mul(df.GrLivArea, axis = 0)
 return X
def counts(df):
X = pd.DataFrame()
X['PorchTypes'] = df[[
'WoodDeckSF',
'OpenPorchSF',
'EnclosedPorch',
'Threeseasonporch',
'ScreenPorch',
]].gt(0.0 ).sum(axis = 1)
 return X
def group_transforms(df):
X = pd.DataFrame()
X['MedNhbdArea'] = df.groupby('Neighborhood')['GrLivArea'].transform('median')
 return X
def break_down(df):
X = pd.DataFrame()
X['MSClass'] = df.MSSubClass.str.split('_', n=1, expand = True)[0]
return X
women = train_data.loc[train_data.Sex == 'female']['Survived']
rate_women = sum(women)/ len(women)
 print('percent of women who survived :', rate_women )<define_variables>
def apply_pca(X, standarize = True):
if standarize:
X =(X - X.mean(axis=0)) / X.std(axis=0)
 pca = PCA()
X_pca = pca.fit_transform(X)
 component_names = [f'PC{i+1}' for i in range(X_pca.shape[1])]
X_pca = pd.DataFrame(X_pca, columns = component_names)
 loadings = pd.DataFrame(pca.components_.T, columns = component_names, index = X.columns)
 return pca, X_pca, loadings
def pca_inspired(df):
X = pd.DataFrame()
X['Feature1'] = df.GrLivArea + df.TotalBsmtSF
X['Feature2'] = df.YearRemodAdd * df.TotalBsmtSF
return X
def pca_components(df, features):
X = df.loc[:, features]
_, X_pca, _ = apply_pca(X)
 return X_pca
pca_features = ['GarageArea', 'YearRemodAdd','TotalBsmtSF','GrLivArea']
men = train_data.loc[train_data.Sex == 'male']["Survived"]
rate_men = sum(men)/len(men)
 print("percent of men who survived :", rate_men )<feature_engineering>
def indicate_outliers(df):
X_new = pd.DataFrame()
X_new['Outlier'] =(df.Neighborhood == 'Edwards')&(df.SaleCondition == 'Partial')
 return X_new
train_data['Family_size'] = train_data['SibSp'] + train_data['Parch'] + 1<count_missing_values>
class CrossFoldEncoder:
def __init__(self,encoder, **kwargs):
self.encoder_ = encoder
self.kwargs_ = kwargs
self.cv_ = KFold(n_splits = 5)
 def fit_transform(self,X,y,cols):
self.fitted_encoders_ = []
self.cols_ = cols
X_encoded = []
for idx_encode, idx_train in self.cv_.split(X):
fitted_encoder = self.encoder_(cols = cols, **self.kwargs_)
 fitted_encoder.fit(X.iloc[idx_encode, :], y.iloc[idx_encode])
 X_encoded.append(fitted_encoder.transform(X.iloc[idx_train, :])[cols])
 self.fitted_encoders_.append(fitted_encoder)
 X_encoded = pd.concat(X_encoded)
 X_encoded.columns = [name + '_encoded' for name in X_encoded.columns]
return X_encoded
def transform(self,X):
X_encoded_list = []
for fitted_encoder in self.fitted_encoders_:
X_encoded = fitted_encoder.transform(X)
 X_encoded_list.append(X_encoded[self.cols_])
 X_encoded = reduce(lambda x,y : x.add(y,fill_value=0), X_encoded_list)/ len(X_encoded_list)
 X_encoded.columns = [name + '_encoded' for name in X_encoded.columns]
return X_encoded
missing_age_value = train_data['Age'].isnull().sum()
percent_of_missing_age_value = missing_age_value / 100
print('% of missing age value =', percent_of_missing_age_value )<data_type_conversions>
def create_features(df, df_test = None):
X = df.copy()
y = X.pop('SalePrice')
 mi_scores = make_mi_scores(X,y)
 if df_test is not None:
X_test = df_test.copy()
X_test.pop('SalePrice')
 X = pd.concat([X, X_test])
 X = drop_uninformative(X, mi_scores)
 X = X.join(mathematical_transforms(X))
X = X.join(interactions(X))
X = X.join(counts(X))
X = X.join(break_down(X))
X = X.join(group_transforms(X))
X = X.join(cluster_labels(X, cluster_features, n_clusters = 20))
X = X.join(pca_inspired(X))
X = label_encode(X)
 if df_test is not None:
X_test = X.loc[df_test.index, :]
X.drop(df_test.index, inplace=True)
 encoder = CrossFoldEncoder(MEstimateEncoder, m=1)
 X = X.join(encoder.fit_transform(X, y, cols= ['MSSubClass']))
if df_test is not None:
X_test = X_test.join(encoder.transform(X_test))
if df_test is not None:
return X, X_test
else:
return X
df_train, df_test = load_data()
X_train = create_features(df_train)
 y_train = df_train.loc[:, 'SalePrice']
score_dataset(X_train, y_train )
train_data['Age_5'] = train_data['Age'] // 5
train_data['Age_5'] = train_data['Age_5'].fillna(mean_age // 5 ).astype('int' )<count_unique_values>
X_train = create_features(df_train)
 y_train = df_train.loc[:, 'SalePrice']
xgb_params = dict(max_depth = 6,
learning_rate = 0.01,
n_estimators = 1000,
min_child_weight = 1,
colsample_bytree = 0.7,
subsample = 0.7,
reg_alpha = 0.5,
reg_lambda = 1,
num_parallel_tree = 1)
 xgb = XGBRegressor(**xgb_params)
 score_dataset(X_train, y_train, xgb )
number_of_unique_age_values = len(train_data['Age_5'].unique())
 print(number_of_unique_age_values )<data_type_conversions>
def objective(trial):
xgb_params = dict(
 max_depth = trial.suggest_int('max_depth',2,10),
learning_rate = trial.suggest_float('learning_rate',1e-4, 1e-1, log=True),
n_estimators = trial.suggest_int('n_estimators',1000,8000),
min_child_weight = trial.suggest_int('min_child_weight', 1,10),
colsample_bytree = trial.suggest_float('colsample_bytree',0.2,1),
subsample = trial.suggest_float('subsample',0.2,1),
reg_alpha = trial.suggest_float('reg_alpha',1e-4, 1e2, log=True),
reg_lambda = trial.suggest_float('reg_lambda', 1e-4, 1e2, log=True))
xgb = XGBRegressor(**xgb_params)
 return score_dataset(X_train, y_train, xgb)
 study = optuna.create_study(direction = 'minimize')
 study.optimize(objective, n_trials = 20)
 xgb_params = study.best_params
<prepare_x_and_y><EOS> 
X_train, X_test = create_features(df_train,df_test)
 y_train = df_train.loc[:, 'SalePrice']
xgb = XGBRegressor(**study.best_params)
 xgb.fit(X_train, np.log(y))
predictions = np.exp(xgb.predict(X_test))
output = pd.DataFrame({'Id': X_test.index, 'SalePrice':predictions})
 output.to_csv('my_submission.csv', index=False)
 print('Your submission was successfully saved!')