Kaggle competition: House Prices - Advanced Regression Techniques
print("
Fitting neural network model...")
 nn.fit(np.array(x_train), np.array(y_train), batch_size = 32, epochs = 70, verbose=2)
 print("
Predicting with neural network model...")
 y_pred_ann = nn.predict(x_test )<drop_column>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 500 )
del train
del prop
del sample
del x_train
del x_test
del df_train
del df_test
del y_pred_ann
gc.collect()<load_from_csv>
print("The number of traning examples(data points)= %i " % train.shape[0])
 print("The number of features we have = %i " % train.shape[1] )
np.random.seed(17)
 random.seed(17)
 print("
Processing data for OLS...")
 train_2017 = pd.read_csv('.. /input/train_2017.csv',parse_dates = ["transactiondate"])
 train_2016 = pd.read_csv('.. /input/train_2016_v2.csv',parse_dates = ["transactiondate"])
 frames = [train_2016,train_2017]
train = pd.concat(frames)
 properties = pd.read_csv(".. /input/properties_2017.csv")
 submission = pd.read_csv(".. /input/sample_submission.csv")
 print(len(train),len(properties),len(submission))
def get_features(df):
df["transactiondate"] = pd.to_datetime(df["transactiondate"])
 df["transactiondate_year"] = df["transactiondate"].dt.year
df["transactiondate_month"] = df["transactiondate"].dt.month
df['transactiondate'] = df['transactiondate'].dt.quarter
df = df.fillna(-1.0)
 return df<compute_test_metric>
train.isnull().sum()
def MAE(y, ypred):
return np.sum([abs(y[i]-ypred[i])for i in range(len(y)) ])/ len(y )<merge>
train.isnull().sum()
train = pd.merge(train, properties, how='left', on='parcelid')
 y = train['logerror'].values
test = pd.merge(submission, properties, how='left', left_on='ParcelId', right_on='parcelid')
 properties = []
exc = [train.columns[c] for c in range(len(train.columns)) if train.dtypes[c] == 'O'] + ['logerror','parcelid']
col = [c for c in train.columns if c not in exc]<feature_engineering>
train.drop(["PoolQC","PavedDrive",'Fence','MiscFeature','FireplaceQu','GarageFinish', 'Alley'], inplace = True, axis = 1)
 test.drop(["PoolQC","PavedDrive",'Fence','MiscFeature','FireplaceQu','GarageFinish' , 'Alley' ], inplace = True, axis = 1 )
train = get_features(train[col])
 test['transactiondate'] = '2016-01-01'
test = get_features(test[col] )<train_model>
train = pd.get_dummies(train)
 test = pd.get_dummies(test )
print("
Fitting OLS...")
 reg = LinearRegression(n_jobs=-1)
 reg.fit(train, y); print('fit...')
 print(MAE(y, reg.predict(train)))
 train = []; y = []
test_dates = ['2016-10-01','2016-11-01','2016-12-01','2017-10-01','2017-11-01','2017-12-01']
test_columns = ['201610','201611','201612','201710','201711','201712']<define_variables>
y = train['SalePrice']
train = train.drop(["SalePrice"], axis = 1 )
print("
Combining XGBoost, LightGBM, NN, and baseline predicitons...")
 lgb_weight = 1 - XGB_WEIGHT - BASELINE_WEIGHT - NN_WEIGHT - OLS_WEIGHT
lgb_weight0 = lgb_weight /(1 - OLS_WEIGHT)
 xgb_weight0 = XGB_WEIGHT /(1 - OLS_WEIGHT)
 baseline_weight0 = BASELINE_WEIGHT /(1 - OLS_WEIGHT)
 nn_weight0 = NN_WEIGHT /(1 - OLS_WEIGHT)
 pred0 = 0
pred0 += xgb_weight0*xgb_pred
pred0 += baseline_weight0*BASELINE_PRED
pred0 += lgb_weight0*p_test
pred0 += nn_weight0*nn_pred<save_to_csv>
train.drop(missing_cols , inplace = True, axis = 1 )
print("
Predicting with OLS and combining with XGB/LGB/NN/baseline predicitons:...")
 for i in range(len(test_dates)) :
test['transactiondate'] = test_dates[i]
pred = FUDGE_FACTOR *(OLS_WEIGHT*reg.predict(get_features(test)) +(1-OLS_WEIGHT)*pred0)
 submission[test_columns[i]] = [float(format(x, '.4f')) for x in pred]
print('predict...', i)
 print("
Writing results to disk...")
 submission.to_csv('1003_try.csv', index=False , float_format='%.4f')
 print("
Finished..." )<feature_engineering>
mean = train.mean().astype(np.int32)
 train.fillna(mean , inplace = True)
 mean = test.mean().astype(np.int32)
 test.fillna(mean , inplace = True )
missing_df = properties.isnull().sum(axis=0 ).reset_index()
missing_df.columns = ['column_name','missing_count']
missing_df['missing_ratio'] = missing_df['missing_count'] / properties.shape[0]
missing_filtered = missing_df.loc[missing_df['missing_ratio']>0.99]
eliminate_list = missing_filtered['column_name'].values
properties = properties.drop(eliminate_list,axis=1 )<data_type_conversions>
train.isnull().sum()
def convert_datatype(dataframe):
for c, dtype in zip(dataframe.columns, dataframe.dtypes):
if dtype == np.float64:
dataframe[c] = dataframe[c].astype(np.float32)
 if dtype == np.int64:
dataframe[c] = dataframe[c].astype(np.int32)
 convert_datatype(properties)
 convert_datatype(test )<feature_engineering>
train.isnull().sum()
properties['living_area_prop'] = properties['calculatedfinishedsquarefeet'] / \
properties['lotsizesquarefeet']
properties['value_ratio'] = properties['taxvaluedollarcnt'] / properties['taxamount']
properties['value_prop'] = properties['structuretaxvaluedollarcnt'] /\
properties['landtaxvaluedollarcnt']<merge>
train['LotFA'] = train['LotArea'] + train['LotFrontage']
train['LotFA2'] = train['LotArea'] - train['LotFrontage']
train['OverallRate'] = train['OverallQual'] + train['OverallCond']
train['OverallRate2'] = train['OverallQual'] - train['OverallCond']
train['yearAvg'] = train['YearBuilt'] + train['YearRemodAdd']
train['yearAvg2'] = train['YearBuilt'] - train['YearRemodAdd']
train['fsFeet'] = train['1stFlrSF'] + train['2ndFlrSF']
train['fsFeet2'] = train['1stFlrSF'] - train['2ndFlrSF']
train['bath'] = train['BsmtFullBath'] + train['BsmtHalfBath']
train['bath2'] = train['BsmtFullBath'] - train['BsmtHalfBath']
train['areaPerCar'] = train['GarageArea'] / train['GarageCars']
train['OpenEnclosedPorch'] = train['OpenPorchSF'] + train['EnclosedPorch']
train['OpenEnclosedPorch2'] = train['OpenPorchSF'] - train['EnclosedPorch']
train['yearSANEG'] =(train['YrSold']**2 - train['yearAvg']**2)**0.5
train['yearSAPOS'] =(train['YrSold']**2 + train['yearAvg']**2)**0.5
test['LotFA'] = test['LotArea'] + test['LotFrontage']
test['LotFA2'] = test['LotArea'] - test['LotFrontage']
test['OverallRate'] = test['OverallQual'] + test['OverallCond']
test['OverallRate2'] = test['OverallQual'] - test['OverallCond']
test['yearAvg'] = test['YearBuilt'] + test['YearRemodAdd']
test['yearAvg2'] = test['YearBuilt'] - test['YearRemodAdd']
test['fsFeet'] = test['1stFlrSF'] + test['2ndFlrSF']
test['fsFeet2'] = test['1stFlrSF'] - test['2ndFlrSF']
test['bath'] = test['BsmtFullBath'] + test['BsmtHalfBath']
test['bath2'] = test['BsmtFullBath'] - test['BsmtHalfBath']
test['areaPerCar'] = test['GarageArea'] / test['GarageCars']
test['OpenEnclosedPorch'] = test['OpenPorchSF'] + test['EnclosedPorch']
test['OpenEnclosedPorch2'] = test['OpenPorchSF'] - test['EnclosedPorch']
test['yearSANEG'] =(test['YrSold']**2 - test['yearAvg']**2)**0.5
test['yearSAPOS'] =(test['YrSold']**2 + test['yearAvg']**2)**0.5
df_train = train.merge(properties, how='left', on='parcelid')
 df_test = test.merge(properties, how='left', on='parcelid' )<categorify>
train.fillna(0 , inplace = True)
 test.fillna(0 , inplace = True )
def convert_label(dataframe):
lbI = LabelEncoder()
for c in dataframe.columns:
dataframe[c] = dataframe[c].fillna(0)
 if dataframe[c].dtype == 'object':
lbI.fit(list(dataframe[c].values))
dataframe[c] = lbI.transform(list(dataframe[c].values))
convert_label(df_train)
 convert_label(df_test)
<prepare_x_and_y>
train.isnull().sum()
x_train = df_train.drop(['parcelid', 'logerror', 'transactiondate', 'propertyzoningdesc',
'propertycountylandusecode', ], axis=1)
 x_test = df_test.drop(['parcelid', 'propertyzoningdesc',
'propertycountylandusecode', '201610', '201611',
'201612', '201710', '201711', '201712'], axis = 1)
 x_train = x_train.values
y_train = df_train['logerror'].values<import_modules>
train.isnull().sum()
from datetime import datetime 
<choose_model_class>
cluster = KMeans(n_clusters= 800, max_iter=300, tol=0.0001, verbose=0, random_state = 0, n_jobs=-1 )
class PAM_Module(nn.Module):

 def __init__(self, in_dim):
super(PAM_Module, self ).__init__()
self.chanel_in = in_dim
self.query_conv = nn.Conv2d(in_channels=in_dim, out_channels=in_dim//8, kernel_size=1)
 self.key_conv = nn.Conv2d(in_channels=in_dim, out_channels=in_dim//8, kernel_size=1)
 self.value_conv = nn.Conv2d(in_channels=in_dim, out_channels=in_dim, kernel_size=1)
 self.gamma = nn.Parameter(torch.zeros(1))
def forward(self, x):

 m_batchsize, C, height, width = x.size()
proj_query = self.query_conv(x ).view(m_batchsize, -1, width*height ).permute(0, 2, 1)
 proj_key = self.key_conv(x ).view(m_batchsize, -1, width*height)
 energy = torch.bmm(proj_query, proj_key)
 attention = torch.softmax(energy, dim=-1)
 proj_value = self.value_conv(x ).view(m_batchsize, -1, width*height)
 out = torch.bmm(proj_value, attention.permute(0, 2, 1))
out = out.view(m_batchsize, C, height, width)
 out = self.gamma*out + x
return out
class CAM_Module(nn.Module):

 def __init__(self, in_dim):
super(CAM_Module, self ).__init__()
self.chanel_in = in_dim
self.gamma = nn.Parameter(torch.zeros(1))
def forward(self,x):

 m_batchsize, C, height, width = x.size()
proj_query = x.view(m_batchsize, C, -1)
 proj_key = x.view(m_batchsize, C, -1 ).permute(0, 2, 1)
 energy = torch.bmm(proj_query, proj_key)
 energy_new = torch.max(energy, -1, keepdim=True)[0].expand_as(energy)-energy
attention = torch.softmax(energy_new, dim=-1)
 proj_value = x.view(m_batchsize, C, -1)
 out = torch.bmm(attention, proj_value)
 out = out.view(m_batchsize, C, height, width)
 out = self.gamma*out + x
return out
class CBAM(nn.Module):
def __init__(self, in_channels):
super(CBAM, self ).__init__()
inter_channels = in_channels // 4
self.conv1_c = nn.Sequential(nn.Conv2d(in_channels, inter_channels, 3, padding=1, bias=False),
nn.BatchNorm2d(inter_channels),
nn.ReLU())
 self.conv1_s = nn.Sequential(nn.Conv2d(in_channels, inter_channels, 3, padding=1, bias=False),
nn.BatchNorm2d(inter_channels),
nn.ReLU())
 self.channel_gate = CAM_Module(inter_channels)
 self.spatial_gate = PAM_Module(inter_channels)
 self.conv2_c = nn.Sequential(nn.Conv2d(inter_channels, in_channels, 3, padding=1, bias=False),
nn.BatchNorm2d(in_channels),
nn.ReLU())
 self.conv2_a = nn.Sequential(nn.Conv2d(inter_channels, in_channels, 3, padding=1, bias=False),
nn.BatchNorm2d(in_channels),
nn.ReLU())
 def forward(self, x):
feat1 = self.conv1_c(x)
 chnl_att = self.channel_gate(feat1)
 chnl_att = self.conv2_c(chnl_att)
 feat2 = self.conv1_s(x)
 spat_att = self.spatial_gate(feat2)
 spat_att = self.conv2_a(spat_att)
 x_out = chnl_att + spat_att
return x_out 
<choose_model_class>
kmeans_train = cluster.fit(train)
 labels_train = kmeans_train.labels_
labels_train
class EffNetWLF(nn.Module):
def __init__(self, model_name, target_size=11):
super().__init__()
self.backbone = EfficientNet.from_name(model_name)
 self.backbone._dropout = nn.Dropout(0.1)
 n_features = self.backbone._fc.in_features
self.backbone._fc = nn.Linear(n_features, target_size)
 self.local_fe = CBAM(n_features)
 self.dropout = nn.Dropout(0.1)
 self.classifier = nn.Sequential(nn.Linear(n_features + n_features, n_features),
nn.BatchNorm1d(n_features),
nn.Dropout(0.1),
nn.ReLU() ,
nn.Linear(n_features, target_size))
def forward(self, image):
enc_feas = self.backbone.extract_features(image)
 global_feas = self.backbone._avg_pooling(enc_feas)
 global_feas = global_feas.flatten(start_dim=1)
 global_feas = self.dropout(global_feas)
 local_feas = self.local_fe(enc_feas)
 local_feas = torch.sum(local_feas, dim=[2,3])
 local_feas = self.dropout(local_feas)
 all_feas = torch.cat([global_feas, local_feas], dim=1)
 outputs = self.classifier(all_feas)
 return outputs 
<load_from_csv>
kmeans_testing = cluster.predict(test)
 kmeans_testing
work_dir = ".. /input/ranzcr-clip-catheter-line-classification/"
df = pd.read_csv(os.path.join(work_dir, "sample_submission.csv"))
test_img_paths = glob.glob(os.path.join(work_dir, "test/*.jpg"))
print(len(test_img_paths))
model_weights = glob.glob(".. /input/effb2wlf/*.pth")
 print(model_weights )<create_dataframe>
train['cluster'] = labels_train
test ['cluster'] = kmeans_testing
normalize = a_transform.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225], p=1.0, max_pixel_value=255.0)
 test_transform = a_transform.Compose([a_transform.Resize(512, 512),
normalize,
ToTensorV2() ], p=1.0)
 test_ds = TestDataset(test_img_paths, test_transform)
 dataloader = DataLoader(test_ds, batch_size=16, num_workers=4, shuffle=False, drop_last=False )<choose_model_class>
print("The number of traning examples(data points)= %i " % train.shape[0])
 print("The number of features we have = %i " % train.shape[1] )
final_pred = np.empty(( len(model_weights),len(test_ds), 11), dtype=np.float32)
 for model_idx, each_w in enumerate(model_weights):
print(f"running idx {model_idx}")
 model = EffNetWLF("efficientnet-b2")
 model = model.to(device)
 checkpoint = torch.load(f"{each_w}")
 model.load_state_dict(checkpoint["model"])
 uids = []
all_pred = []
for idx,(name, x_mb)in enumerate(dataloader):
x = x_mb.to(device)
 x = torch.stack([x,x.flip(-1)],0)
 x = x.view(-1, 3, 512, 512)
 with torch.no_grad() :
pred = model(x)
 pred = pred.view(2, x_mb.size(0), -1 ).mean(0)
 uids.append(name)
 all_pred.append(torch.sigmoid(pred))
all_pred = torch.cat(all_pred, dim=0 ).cpu().numpy()
final_pred[model_idx] = all_pred
final_pred = final_pred.mean(axis=0)
 print(final_pred.shape)
 target_cols = df.columns[1:]
df[target_cols] = final_pred
df["StudyInstanceUID"] = np.concatenate(uids)
 df.to_csv('submission.csv', index=False)
 df.head()<set_options>
id = test['Id']
test.drop(['Id'], axis = 1, inplace = True)
 train.drop(['Id'], axis = 1, inplace = True)
sys.path.append('.. /input/pytorch-images-seresnet')
 warnings.filterwarnings('ignore')
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )<define_variables>
x_train, x_test, y_train, y_test = train_test_split(train.values , y.values, test_size=0.05, random_state=42 )
IMAGE_SIZE = 640
BATCH_SIZE = 64
TEST_PATH = '.. /input/ranzcr-clip-catheter-line-classification/test'
MODEL_PATH = '.. /input/efficientnetb5cv9621/tf_efficientnet_b5_ns_CV96.21.pth'<load_from_csv>
et = ExtraTreesRegressor(n_estimators=950 , max_features = 'auto', max_leaf_nodes=None, n_jobs= -1, random_state = 0, verbose = 0)
 gbr = GradientBoostingRegressor()
lasso = Lasso()
xgbr = XGBRegressor()
svr = SVR(kernel= 'rbf', gamma= 'auto', tol=0.001, C=100.0, max_iter=-1)
 rf = RandomForestRegressor(n_estimators=900, random_state=0)
 lr = LinearRegression(fit_intercept=True, normalize=True, copy_X=True, n_jobs=-1)
 knnR = KNeighborsRegressor(n_neighbors=20, n_jobs=-1)
 reg = StackingCVRegressor(regressors=[ lasso , xgbr , et],meta_regressor=lasso)
 reg.fit(x_train, y_train, groups = None)
test = pd.read_csv('.. /input/ranzcr-clip-catheter-line-classification/sample_submission.csv' )<categorify>
reg.score(x_test,y_test )
def get_transforms() :
return Compose([
Resize(IMAGE_SIZE, IMAGE_SIZE),
Normalize(
),
ToTensorV2() ,
] )<choose_model_class>
sub = pd.DataFrame({"Id": id ,"SalePrice":(reg.predict(test.values)).round(decimals=2)})
 sub.to_csv("stackcv_linearsvc.csv", index=False )