Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<create_dataframe>
warnings.filterwarnings("ignore" )
df_total_sales_customers = pd.DataFrame({'Sales': total_sales_customers['Sales'],
'Customers': total_sales_customers['Customers']},
index = total_sales_customers.index)
 df_total_sales_customers = df_total_sales_customers.reset_index()
df_total_sales_customers.head()<groupby>
test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test["SalePrice"] = np.nan
avg_sales_customers = df_train.groupby('Store')['Sales', 'Customers'].mean()
avg_sales_customers.head()<merge>
num_discrete = ['BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr','TotRmsAbvGrd',
'Fireplaces','GarageCars','GarageYrBlt','YearBuilt','YearRemodAdd','YrSold','MoSold']
num_continuous = ['LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','1stFlrSF',
'2ndFlrSF','LowQualFinSF','GrLivArea','GarageArea','WoodDeckSF','OpenPorchSF','EnclosedPorch',
'3SsnPorch','ScreenPorch','PoolArea','MiscVal','SalePrice']
cat_ordinal = ['LotShape','Utilities','LandSlope','OverallQual','OverallCond','ExterQual','ExterCond',
'BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','HeatingQC','CentralAir',
'Electrical','KitchenQual','Functional','FireplaceQu','GarageFinish','GarageQual','GarageCond',
'PavedDrive','PoolQC','Fence']
cat_nominal = ['MSSubClass','MSZoning','Street','Alley','LandContour','LotConfig','Neighborhood',
'Condition1','Condition2','BldgType','HouseStyle','RoofStyle','RoofMatl','Exterior1st',
'Exterior2nd','MasVnrType','Foundation','Heating','GarageType','MiscFeature',
'SaleType','SaleCondition']
df_avg_sales_customers = pd.DataFrame({'Sales': avg_sales_customers['Sales'],
'Customers': avg_sales_customers['Customers']},
index = avg_sales_customers.index)
 df_avg_sales_customers = df_avg_sales_customers.reset_index()
df_stores_avg = df_avg_sales_customers.join(df_store.set_index('Store'), on='Store')
 df_stores_avg.head() 
<merge>
def z_score(df):
return(df-df.mean())/df.std(ddof=0 )
df_stores_new = df_total_sales_customers.join(df_store.set_index('Store'), on='Store')
 df_stores_new.head()<groupby>
idx = []
outliers = []
corrs = []
zeros = []
for i in num_continuous:
if str(train[i].dtype)!= 'object':
idx.append(i)
 outliers.append(list(abs(z_score(train[i])) > 3 ).count(True))
corrs.append(train.SalePrice.corr(train[i]))
zeros.append(len(train[i][train[i] == 0])/len(train[i]))
outs = pd.DataFrame({'
'Feature': idx,
'Corr': corrs,
'% Zeros': zeros} ).sort_values(ascending = False,
by = '
outs = outs[outs["
outs = outs[abs(outs["Corr"])> 0.3].reset_index(drop=True)
 outs = outs[outs["% Zeros"] < 0.3].reset_index(drop=True)
 outs
average_storetype = df_stores_new.groupby('StoreType')['Sales', 'Customers', 'CompetitionDistance'].mean()
fig,(axis1,axis2,axis3)= plt.subplots(1,3,figsize=(15,4))
sns.barplot(average_storetype.index, average_storetype['Sales'], ax=axis1)
 sns.barplot(average_storetype.index, average_storetype['Customers'], ax=axis2)
 sns.barplot(average_storetype.index, average_storetype['CompetitionDistance'], ax=axis3)
 average_storetype.index<load_from_csv>
train = train.drop(train[(train['GrLivArea'] > 4000)&(train['SalePrice'] < 300000)].index )
df_train = pd.read_csv(".. /input/train.csv")
 df_store = pd.read_csv(".. /input/store.csv")
 df_test = pd.read_csv(".. /input/test.csv")
<feature_engineering>
train = train.drop(train[(train['LotFrontage'] > 300)&(train['SalePrice'] < 300000)].index )
closed_store_ids = df_test["Id"][df_test["Open"] == 0].values
df_train['Year'] = df_train['Date'].apply(lambda x: int(x[:4]))
df_train['Month'] = df_train['Date'].apply(lambda x: int(x[5:7]))
df_train["HolidayBin"] = df_train.StateHoliday.map({"0": 0, "a": 1, "b": 1, "c": 1})
 del df_train['Date']
del df_train['StateHoliday']
df_test['Year'] = df_test['Date'].apply(lambda x: int(x[:4]))
df_test['Month'] = df_test['Date'].apply(lambda x: int(x[5:7]))
df_test["HolidayBin"] = df_test.StateHoliday.map({"0": 0, "a": 1, "b": 1, "c": 1})
 del df_test['Date']
del df_test['StateHoliday']
df_test.head()<feature_engineering>
dataset = pd.concat([train,test],axis=0 ).reset_index(drop=True)
 dataset = dataset.fillna(np.nan )
df_test = df_test[df_test["Open"] != 0]
df_test[df_test['Store'] == 1].head()
a = list()
for i in df_test['Store']:
a.append(float(df_store['CompetitionDistance'][df_store['Store'] == i]))
df_test['CompetitionDistance'] = a 
<feature_engineering>
def show_null(df):
null_columns =(df.isnull().sum(axis = 0)/len(df)).sort_values(ascending=False ).index
null_data = pd.concat([df.isnull().sum(axis = 0), 
(df.isnull().sum(axis = 0)/len(df)).sort_values(ascending=False),
df.loc[:, df.columns.isin(list(null_columns)) ].dtypes]
, axis=1)
 null_data = null_data.rename(columns={0: '
1: '% null',
2: 'type'} ).sort_values(ascending=False, by = '% null')
 null_data = null_data[null_data["
return null_data
show_null(dataset )
a = list()
for i in df_train['Store']:
a.append(float(df_store['CompetitionDistance'][df_store['Store'] == i]))
df_train['CompetitionDistance'] = a
df_train.head()
df_train['CompetitionDistance'] = df_train['CompetitionDistance'].fillna(df_train['CompetitionDistance'].mean() )<feature_engineering>
dataset.PoolQC[dataset.PoolQC.isnull() == True] = 'NA'
dataset.Fence[dataset.Fence.isnull() == True] = 'NA'
dataset.FireplaceQu[dataset.FireplaceQu.isnull() == True] = 'NA'
dataset.GarageCond[dataset.GarageCond.isnull() == True] = 'NA'
dataset.GarageQual[dataset.GarageQual.isnull() == True] = 'NA'
dataset.GarageFinish[dataset.GarageFinish.isnull() == True] = 'NA'
dataset.BsmtExposure[dataset.BsmtExposure.isnull() == True] = 'NA'
dataset.BsmtCond[dataset.BsmtCond.isnull() == True] = 'NA'
dataset.BsmtQual[dataset.BsmtQual.isnull() == True] = 'NA'
dataset.BsmtFinType2[dataset.BsmtFinType2.isnull() == True] = 'NA'
dataset.BsmtFinType1[dataset.BsmtFinType1.isnull() == True] = 'NA'
dataset.Electrical[dataset.Electrical.isnull() == True] = stats.mode(train.Electrical)[0][0]
dataset.Functional[dataset.Functional.isnull() == True] = stats.mode(train.Functional)[0][0]
dataset.KitchenQual[dataset.KitchenQual.isnull() == True] = stats.mode(train.KitchenQual)[0][0]
dataset.Utilities[dataset.Utilities.isnull() == True] = stats.mode(train.Utilities)[0][0]
df_train['CompetitionDistance'] = np.log(df_train['CompetitionDistance'])
 df_test['CompetitionDistance'] = np.log(df_test['CompetitionDistance'] )<categorify>
dataset.MiscFeature[dataset.MiscFeature.isnull() == True] = 'NA'
dataset.Alley[dataset.Alley.isnull() == True] = 'NA'
dataset.GarageType[dataset.GarageType.isnull() == True] = 'NA'
dataset.MasVnrType[dataset.MasVnrType.isnull() == True] = 'NA'
dataset.MSZoning[dataset.MSZoning.isnull() == True] = stats.mode(train.MSZoning)[0][0]
dataset.SaleType[dataset.SaleType.isnull() == True] = stats.mode(train.SaleType)[0][0]
dataset.Exterior1st[dataset.Exterior1st.isnull() == True] = stats.mode(train.Exterior1st)[0][0]
dataset.Exterior2nd[dataset.Exterior2nd.isnull() == True] = stats.mode(train.Exterior2nd)[0][0]
train_stores = dict(list(df_train.groupby('Store')))
 test_stores = dict(list(df_test.groupby('Store')))
 submission = pd.Series()
for i in test_stores:
store = train_stores[i]
X_train = store.drop(["Sales", "Store", "Customers"],axis=1)
 Y_train = store["Sales"]
X_test = test_stores[i].copy()
store_ids = X_test["Id"]
X_test.drop(["Id","Store"], axis=1,inplace=True)
 X_train = X_train.fillna(X_train.mean())
 X_test = X_test.fillna(X_train.mean())
 rfr = RandomForestRegressor(n_estimators = 5, criterion = 'mse')
 rfr.fit(X_train, Y_train)
 Y_pred = rfr.predict(X_test)
 submission = submission.append(pd.Series(Y_pred, index=store_ids))
submission = submission.append(pd.Series(0, index=closed_store_ids))
submission = pd.DataFrame({ "Id": submission.index, "Sales": submission.values})
 submission.to_csv('rossmann_submission.csv', index=False)
 print(len(submission))<set_options>
dataset.BsmtFullBath[dataset.BsmtFullBath.isnull() == True] = stats.mode(train.BsmtFullBath)[0][0]
dataset.BsmtHalfBath[dataset.BsmtHalfBath.isnull() == True] = stats.mode(train.BsmtHalfBath)[0][0]
dataset.GarageCars[dataset.GarageCars.isnull() == True] = stats.mode(train.GarageCars)[0][0]
dataset.GarageArea[dataset.GarageArea.isnull() == True] = np.median(train.
 GarageArea[train.GarageArea.isnull() == False])
 dataset.TotalBsmtSF[dataset.TotalBsmtSF.isnull() == True] = np.median(train.
 TotalBsmtSF[train.TotalBsmtSF.isnull() == False])
 dataset.BsmtFinSF1[dataset.BsmtFinSF1.isnull() == True] = np.median(train.
 BsmtFinSF1[train.BsmtFinSF1.isnull() == False])
 dataset.BsmtFinSF2[dataset.BsmtFinSF2.isnull() == True] = np.median(train.
 BsmtFinSF2[train.BsmtFinSF2.isnull() == False])
 dataset.BsmtUnfSF[dataset.BsmtUnfSF.isnull() == True] = np.median(train.
 BsmtUnfSF[train.BsmtUnfSF.isnull() == False])
 dataset.LotFrontage[dataset.LotFrontage.isnull() == True] = np.median(train.
 LotFrontage[train.LotFrontage.isnull() == False] )
plt.style.use('ggplot')
 plt.rcParams['figure.figsize'] =(12,8)
 %matplotlib inline<load_from_csv>
dataset[dataset.MasVnrArea.isnull() == True].MasVnrType
df_stores = pd.read_csv('.. /input/store.csv', sep=',')
 df_data = pd.read_csv('.. /input/train.csv', sep=',')
 df_test = pd.read_csv('.. /input/test.csv', sep=',' )<drop_column>
dataset.MasVnrArea[dataset.MasVnrArea.isnull() == True] = 0
df_stores = df_stores.drop('CompetitionOpenSinceMonth', axis=1 ).drop('CompetitionOpenSinceYear', axis=1 ).drop('Promo2SinceWeek', axis=1 ).drop('Promo2SinceYear', axis=1 )<filter>
dataset[["GarageType",
"GarageFinish",
"GarageQual",
"GarageCond",
"GarageYrBlt"]][dataset.GarageYrBlt.isnull() == True].dropna()
closed_stores = df_test["Id"][df_test["Open"] == 0].values
df_test = df_test[df_test["Open"] != 0]
df_data = df_data[df_data["Open"] != 0]<drop_column>
dataset.GarageYrBlt[dataset.GarageYrBlt.isnull() == True] = 'NA'
df_data = df_data.drop('Open', axis=1 ).drop('Customers', axis=1)
 df_test = df_test.drop('Open', axis=1 )<feature_engineering>
num_discrete = ['BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr','TotRmsAbvGrd',
'Fireplaces','GarageCars','GarageYrBlt','YearBuilt','YearRemodAdd','YrSold','MoSold']
df_data['Month'] = df_data['Date'].apply(lambda x: int(x[5:7]))
df_test['Month'] = df_test['Date'].apply(lambda x: int(x[5:7]))
df_data['Year'] = df_data['Date'].apply(lambda x: int(x[:4]))
df_test['Year'] = df_test['Date'].apply(lambda x: int(x[:4]))
df_test = df_test.drop('Date', axis=1)
 df_data = df_data.drop('Date', axis=1 )<merge>
dataset["Baths"] = dataset.BsmtFullBath + 0.5*dataset.BsmtHalfBath + dataset.FullBath + 0.5*dataset.HalfBath
df_stores = df_stores.join(df_data.groupby('Store')['Sales'].mean() , on='Store' ).rename(columns={'Sales': 'Av_sales'} )<feature_engineering>
dataset.Baths = dataset.Baths.apply(lambda x: 3.5 if x > 3.5 else x )
max_s = df_stores['Av_sales'].max()
df_stores['Av_sales']=df_stores['Av_sales']/max_s<merge>
dataset.BedroomAbvGr = dataset.BedroomAbvGr.apply(lambda x: 5 if x > 5 else x )
df_test = pd.merge(df_test, df_stores, left_index=True, on='Store')
 df_data = pd.merge(df_data, df_stores, left_index=True, on='Store' )<correct_missing_values>
dataset.BedroomAbvGr = dataset.BedroomAbvGr.astype(str )
df_data['CompetitionDistance'].fillna(df_data['CompetitionDistance'].median() , inplace = True)
 df_test['CompetitionDistance'].fillna(df_data['CompetitionDistance'].median() , inplace = True )<feature_engineering>
dataset.KitchenAbvGr = dataset.KitchenAbvGr.apply(lambda x: 1 if x==0 else(2 if x==3 else x))
max_dist = df_data['CompetitionDistance'].max()
df_data['CompetitionDistance']=df_data['CompetitionDistance']/max_dist
df_test['CompetitionDistance']=df_test['CompetitionDistance']/max_dist<correct_missing_values>
dataset.KitchenAbvGr = dataset.KitchenAbvGr.astype(str )
df_data['PromoInterval'].fillna('n', inplace = True)
 df_test['PromoInterval'].fillna('n', inplace = True )<feature_engineering>
dataset.TotRmsAbvGrd = dataset.TotRmsAbvGrd.apply(lambda x: 3 if x==2 else(11 if x>11 else x))
df_data["StateHoliday"].loc[df_data["StateHoliday"] == 0] = "0"
df_test["StateHoliday"].loc[df_test["StateHoliday"] == 0] = "0"<count_missing_values>
dataset.GarageCars = dataset.GarageCars.apply(lambda x: 3 if x>3 else x )
df_test.isnull().any()<categorify>
dataset.GarageYrBlt[dataset.GarageYrBlt == 2207] = 2007
features = ['DayOfWeek', 'Month', 'Promo', 'StateHoliday', 'SchoolHoliday',
'StoreType', 'Assortment', 'PromoInterval', 'Year']
for fea in features:
av_sales = df_data.groupby(fea)['Sales'].mean().values
num_dict = dict(df_data.groupby(fea)['Sales'].mean() /max(av_sales))
df_data[fea] = df_data[fea].map(num_dict)
 df_test[fea] = df_test[fea].map(num_dict)
 print(fea, num_dict, '
' )<filter>
min(dataset.GarageYrBlt[dataset["GarageYrBlt"] != 'NA'].values )
df_data = df_data[df_data['Sales']!=0]<filter>
bins = [1890, 1920, 1940, 1950, 1960, 1970, 1980, 1990, 2000, 2011]
names = ['-1920', '1920-1940', '1940-1950', '1950-1960', '1960-1970',
'1970-1980', '1980-1990', '1990-2000', '2000+']
dataset.GarageYrBlt[dataset["GarageYrBlt"] != 'NA'] = pd.cut(dataset.GarageYrBlt[dataset["GarageYrBlt"] != 'NA'],
bins,
labels=names )
df_data = df_data[df_data['Sales']<34500]<drop_column>
dataset.GarageYrBlt = dataset.GarageYrBlt.astype(str )
df_data = df_data.drop('Store', axis=1)
 df_test = df_test.drop('Store', axis=1 )<split>
bins = [1949, 1960, 1970, 1980, 1990, 2000, 2011]
names = ['1950-1960', '1960-1970',
'1970-1980', '1980-1990', '1990-2000', '2000+']
dataset.YearRemodAdd = pd.cut(dataset.YearRemodAdd, bins, labels=names )
df_train, df_mytest = train_test_split(df_data, test_size = 0.2 )<prepare_x_and_y>
dataset.YearRemodAdd = dataset.YearRemodAdd.astype(str )
X_train, y_train = df_train.drop('Sales', axis=1 ).values, df_train['Sales'].values<choose_model_class>
dataset.YearBuilt = dataset.YearBuilt.astype(str )
model = LinearRegression(n_jobs=4 )<train_model>
dataset.YrSold = dataset.YrSold.astype(str )
model.fit(X_train, y_train )<train_model>
dataset.MoSold = dataset.MoSold.astype(str )
model.fit(X_train, y_train )<compute_test_metric>
num_continuous = ['LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF',
'1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea','GarageArea','WoodDeckSF','OpenPorchSF',
'EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','MiscVal']
def eval_rmspe(y_hat, y_true):
res = np.sqrt(sum(((y_true-y_hat)/y_true)**2)/len(y_true))
return res<prepare_x_and_y>
dataset.LotFrontage = boxcox1p(dataset.LotFrontage, 0.35 )
X_test, y_mytest = df_mytest.drop('Sales', axis=1 ).values, df_mytest['Sales'].values<predict_on_test>
dataset.LotArea = boxcox1p(dataset.LotArea, 0.15 )
y_hat = model.predict(X_test )<compute_test_metric>
dataset.MasVnrArea = boxcox1p(dataset.MasVnrArea, 0.15 )
print(eval_rmspe(y_hat, y_mytest))<import_modules>
dataset.BsmtFinSF1 = dataset.BsmtFinSF1.astype(str )
from sklearn.linear_model import Ridge<train_model>
dataset.BsmtFinSF2 = dataset.BsmtFinSF2.apply(lambda x: 1 if x > 0 else 0 )
R_model = Ridge(alpha=1)
 R_model.fit(X_train, y_train )<predict_on_test>
dataset.BsmtUnfSF = dataset.BsmtUnfSF.astype(str )
y_hat_R = R_model.predict(X_test)
 print(eval_rmspe(y_hat_R, y_mytest))<import_modules>
dataset["FlrSF"] = dataset["1stFlrSF"] + dataset["2ndFlrSF"] + dataset["TotalBsmtSF"]
from sklearn.ensemble import RandomForestRegressor<prepare_x_and_y>
dataset.FlrSF = boxcox1p(dataset.FlrSF, 0.35)
 dataset["1stFlrSF"] = boxcox1p(dataset["1stFlrSF"], 0.15 )
X_train_full, y_train_full = df_data.drop('Sales', axis=1 ).values, df_data['Sales'].values<drop_column>
dataset['2ndFlrSF'] = dataset['2ndFlrSF'].astype(str )
X_test_full = df_test.drop('Id', axis=1 ).values<train_model>
dataset.LowQualFinSF = dataset.LowQualFinSF.apply(lambda x: 1 if x > 0 else 0 )
forest = RandomForestRegressor(n_jobs=1, n_estimators=150, max_features=7, max_depth=100)
 forest.fit(X_train_full, y_train_full )<predict_on_test>
dataset.GrLivArea = boxcox1p(dataset.GrLivArea,0.15 )
y_hat_full = forest.predict(X_test_full )<define_variables>
dataset.WoodDeckSF = boxcox1p(dataset.WoodDeckSF,0.15 )
store_ids = df_test['Id'].values
store_ids<save_to_csv>
dataset["PorchSF"] = dataset["OpenPorchSF"] + dataset["EnclosedPorch"] + dataset["3SsnPorch"] + dataset["ScreenPorch"]
dataset = dataset.drop(columns=["OpenPorchSF","EnclosedPorch","3SsnPorch","ScreenPorch"] )
submission = submission.append(pd.Series(y_hat_full, index=store_ids))
submission = pd.DataFrame({ "Id": submission.index, "Sales": submission.values})
 submission.to_csv('submission.csv', index=False )<set_options>
dataset.PorchSF = boxcox1p(dataset.PorchSF, 0.35 )
def metric(y_true, y_pred):
return np.mean(np.sum(np.abs(y_true - y_pred), axis=0)/np.sum(y_true, axis=0))
display(HTML("<style>.container { width:100% !important; }</style>"))
base_path = '/kaggle/input/trends-assessment-prediction/'
feature_path = '/kaggle/input/pni-trends-features/'<load_from_csv>
dataset = dataset.drop(columns=["MiscVal"] )
prediction_pooled = pd.read_csv(feature_path + "site_prediction.csv" ).drop('Unnamed: 0', axis=1)
 graphparams = pd.read_csv(feature_path + 'fnc_graph_params.csv')
 csf_df = pd.read_csv(feature_path + 'csf_features')
 wm_df = pd.read_csv(feature_path + 'wm_features')
 gm_all_df = pd.read_csv(feature_path + 'gm_all_features')
 gm_df = pd.read_csv(feature_path + 'gm_max_features')
 pni_df = pd.read_csv(feature_path + 'PNI_features')
 loading_df = pd.read_csv(base_path +"loading.csv")
 asym_df = pd.read_csv(feature_path + 'asym_winsorized_features' )<create_dataframe>
cat_ordinal = ['LotShape','Utilities','LandSlope','OverallQual','OverallCond','ExterQual','ExterCond',
'BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','HeatingQC','CentralAir',
'Electrical','KitchenQual','Functional','FireplaceQu','GarageFinish','GarageQual','GarageCond',
'PavedDrive','PoolQC','Fence']
fnc_data = pd.read_csv(base_path + 'fnc.csv')

networks = np.array(['SCN', 'ADN', 'SMN', 'VSN', 'CON', 'DMN', 'CBN'])
 internal_conn = np.zeros(( len(fnc_data.Id), len(networks)))
 external_conn = np.zeros(( len(fnc_data.Id), len(networks)))
 netmat = pd.DataFrame({'Id': fnc_data.Id})
 netmat_count = pd.DataFrame({'Id': [int(1)]})
 internal_count = np.zeros(len(networks))
external_count = np.zeros(len(networks))
for c in fnc_data.drop('Id', axis=1 ).columns:
splitted=c.split('_')
 netmat_col=splitted[0][:3] + '_vs_' + splitted[2][:3]
if netmat_col not in netmat:
netmat[netmat_col]=fnc_data[c].values
netmat_count[netmat_col]=1
else:
netmat[netmat_col]+=fnc_data[c].values
netmat_count[netmat_col]+=1
if splitted[0][:3] == splitted[2][:3]:
whichnetwork = np.argwhere(networks==splitted[0][:3])[0][0]
internal_conn[:,whichnetwork] += fnc_data[c].values
internal_count[whichnetwork] += 1
else:
whichnetwork1 = np.argwhere(networks==splitted[0][:3])[0][0]
external_conn[:,whichnetwork1] += fnc_data[c].values
whichnetwork2 = np.argwhere(networks==splitted[2][:3])[0][0]
external_conn[:,whichnetwork2] += fnc_data[c].values
external_count[whichnetwork1] += 1
external_count[whichnetwork2] += 1
netmat /= netmat_count.values
netmat = pd.DataFrame(netmat)
 internal_conn /= internal_count
external_conn /= external_count
internal_conn = pd.DataFrame(internal_conn, columns=[ "int_" + str(c)for c in networks])
 internal_conn.insert(0, 'Id', fnc_data.Id)
 external_conn = pd.DataFrame(external_conn, columns=[ "ext_" + str(c)for c in networks])
 external_conn.insert(0, 'Id', fnc_data.Id)
 internal_conn_tmp = internal_conn.copy()
external_conn_tmp = external_conn.copy()
external_conn_tmp.columns = internal_conn_tmp.columns = ['Id'] + ["div_" + str(c)for c in networks]
intdivext = internal_conn_tmp/external_conn_tmp
intdivext.Id=internal_conn.Id
external_conn_tmp.columns = internal_conn_tmp.columns = ['Id'] + ["mult_" + str(c)for c in networks]
intmultext = internal_conn_tmp*external_conn_tmp
intmultext.Id=internal_conn.Id 
<load_from_csv>
dict_ = {"PoolQC": {"NA": 0, "Fa": 1, "TA": 2, "Gd": 3, "Ex": 4},
"Fence": {"NA": 0, "MnWw": 1, "GdWo": 2, "MnPrv": 3, "GdPrv": 4},
"FireplaceQu": {"NA": 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5},
"GarageCond": {"NA": 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5},
"GarageQual": {"NA": 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5},
"GarageFinish": {"NA": 0, "Unf": 1, "RFn": 2, "Fin": 3},
"BsmtExposure": {"NA": 0, "No": 1, "Mn": 2, "Av": 3, "Gd": 4},
"BsmtCond": {"NA": 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5},
"BsmtQual": {"NA": 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5},
"BsmtFinType2": {"NA": 0, "Unf": 1, "LwQ": 2, "Rec": 3, "BLQ": 4, "ALQ": 5, "GLQ": 6},
"BsmtFinType1": {"NA": 0, "Unf": 1, "LwQ": 2, "Rec": 3, "BLQ": 4, "ALQ": 5, "GLQ": 6},
"Electrical": {"Mix": 0, "FuseP": 1, "FuseF": 2, "FuseA": 3, "SBrkr": 4},
"Functional": {"Sal": 0, "Sev": 1, "Maj2": 2, "Maj1": 3, "Mod": 4, "Min2": 5, "Min1": 6, "Typ": 7},
"KitchenQual": {"Po": 0, "Fa": 1, "TA": 2, "Gd": 3, "Ex": 4},
"Utilities": {"ELO": 0, "NoSeWa": 1, "NoSewr": 2, "AllPub": 3},
"LotShape": {"IR3": 0, "IR2": 1, "IR1": 2, "Reg": 3},
"LandSlope": {"Sev": 0, "Mod": 1, "Gtl": 2},
"ExterQual": {"Po": 0, "Fa": 1, "TA": 2, "Gd": 3, "Ex": 4},
"ExterCond": {"Po": 0, "Fa": 1, "TA": 2, "Gd": 3, "Ex": 4},
"HeatingQC": {"Po": 0, "Fa": 1, "TA": 2, "Gd": 3, "Ex": 4},
"CentralAir": {"N": 0, "Y": 1},
"FireplaceQu": {"NA": 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5},
"PavedDrive": {"N": 0, "P": 1, "Y": 2}}
dataset.replace(dict_, inplace=True )
fnc=fnc_data
fnc_feat_vecs = fnc.iloc[:, 1:].values
fnc_feat_vecs.shape
icn_numbers = pd.read_csv(base_path + "ICN_numbers.csv")
 icn_numbers["FNC_name"] = ["SCN(69)"] + list(map(lambda x: x.split("_")[0], fnc.columns[1:53]))
col_labels = icn_numbers["FNC_name"].values
degree=pd.DataFrame(np.zeros(( fnc_feat_vecs.shape[0], len(col_labels))), columns=['degree_' + c for c in col_labels])
 clustering_r=pd.DataFrame(np.zeros(( fnc_feat_vecs.shape[0], len(col_labels))), columns=['clusteringr_' + c for c in col_labels])
 clustering_i=pd.DataFrame(np.zeros(( fnc_feat_vecs.shape[0], len(col_labels))), columns=['clusteringi_' + c for c in col_labels])
 betweenness_centrality=pd.DataFrame(np.zeros(( fnc_feat_vecs.shape[0], len(col_labels))), columns=['between_' + c for c in col_labels])
 eigenvector_centrality=pd.DataFrame(np.zeros(( fnc_feat_vecs.shape[0], len(col_labels))), columns=['eigenvec_' + c for c in col_labels])
<find_best_model_class>
dataset.BsmtCond = dataset.BsmtCond.astype(int )
df_pandas = df.copy()
tmp_df=prediction_pooled.merge(df_pandas, on='Id')
 for c in tmp_df.columns:
if c=='Id' or c=='site_predicted_fixed' or c=='site_predicted' or c=='predicted':
continue
model = LinearRegression()
model.fit(tmp_df.site_predicted_fixed.values.reshape(-1,1), tmp_df[c].values)
 prediction = model.predict(tmp_df.site_predicted_fixed.values.reshape(-1,1))
df_pandas.at[:,c] =(df_pandas[c] - prediction)
 df_sitereg = pd.DataFrame(df_pandas)
 df_sitereg = df_sitereg.merge(gm_all_df , on = 'Id')
 labels_df = pd.read_csv(base_path + "train_scores.csv")
 labels_df["is_train"] = True
df = df_sitereg.merge(labels_df, on="Id", how="left")
 test_df = df[df["is_train"] != True].copy()
df = df[df["is_train"] == True].copy()<normalization>
dataset.LotShape = dataset.LotShape.apply(lambda x: 1 if x == 0 else x )
with open(feature_path + 'optpars.json', 'r')as fp:
optpars = json.load(fp)
 def down_scale(x, factor):
return x*factor
def square(x):
return x*x
def winsorize(x, l=0.0001, h=0.0001):
return scipy.stats.mstats.winsorize(x,(l, h))<define_variables>
dataset = dataset.drop(columns='Utilities' )
test_result=test_df.copy()
reg = dict()
features = col_loading_features+col_fnc_features+col_netmat_features+col_internalconn_features+col_externalconn_features+col_intmultext_features+col_gm_features+col_wm_features+col_csf_features+col_asym_features+col_graphparams_features+col_gm_all+col_pni
for target in ['age', 'domain1_var1', 'domain1_var2', 'domain2_var1', 'domain2_var2']:
print("*****",target)
 dfnona = df[df[target].notnull() ]
estimators = [ 
('sbm+funcdeweighted_win_ridge',
Pipeline([('columns', ColumnTransformer([ 
('sbm', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_loading_features), 
('fnc', FunctionTransformer(down_scale, kw_args={'factor': 1/600}, validate=True), col_fnc_features), 
('internal_conn', FunctionTransformer(down_scale, kw_args={'factor': 1/600}, validate=True), col_internalconn_features), 
('external_conn', FunctionTransformer(down_scale, kw_args={'factor': 1/600}, validate=True), col_externalconn_features), 
('intmultext', FunctionTransformer(down_scale, kw_args={'factor': 1/600}, validate=True), col_intmultext_features),
],remainder='drop')) , 
('win', FunctionTransformer(winsorize, kw_args={'l': 0.0001, 'h': 0.0001}, validate=True)) , 
('e', Ridge(alpha=optpars['sbm+funcdeweighted_win_ridge'][target]))
])) , 
('sbm+funcdeweighted_win_ridgebag',
Pipeline([('columns', ColumnTransformer([ 
('sbm', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_loading_features), 
('fnc', FunctionTransformer(down_scale, kw_args={'factor': 1/600}, validate=True), col_fnc_features), 
('gm', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_gm_features), 
('gm2', FunctionTransformer(square, validate=True), col_gm_features), 
('wm', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_wm_features), 
('wm2', FunctionTransformer(square, validate=True), col_wm_features), 
('csf', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_csf_features), 
('csf2', FunctionTransformer(square, validate=True), col_csf_features),
],remainder='drop')) , 
('win', FunctionTransformer(winsorize, kw_args={'l': 0.0001, 'h': 0.0001}, validate=True)) , 
('e', BaggingRegressor(Ridge(alpha=optpars['sbm+funcdeweighted_win_ridgebag'][target]),
n_estimators=20,
random_state=42,
max_samples=0.8,
max_features=0.8))
])) , 
('sbm+sbm_win_ridge',
Pipeline([('columns', ColumnTransformer([('sbm', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_loading_features), 
('sbm2', FunctionTransformer(square, validate=True), col_loading_features)
 ],remainder='drop')) , 
('win', FunctionTransformer(winsorize, kw_args={'l': 0.0001, 'h': 0.0001}, validate=True)) , 
('e', Ridge(alpha=optpars['sbm+sbm_win_ridge_alpha'][target]))
])) , 
('sbm+sbm_win_ridgebag',
Pipeline([('columns', ColumnTransformer([('sbm', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_loading_features), 
('sbm2', FunctionTransformer(square, validate=True), col_loading_features)
 ],remainder='drop')) , 
('win', FunctionTransformer(winsorize, kw_args={'l': 0.0001, 'h': 0.0001}, validate=True)) , 
('e', BaggingRegressor(Ridge(alpha=optpars['sbm+sbm_win_ridge_alphabag'][target]),
n_estimators=20,
random_state=42,
max_samples=0.8,
max_features=0.8))
])) , 
('fnc_extended_ridge', Pipeline([ 
('columns', ColumnTransformer([('fnc', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_fnc_features), 
('internal_conn', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_internalconn_features), 
('external_conn', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_externalconn_features), 
('intmultext', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_intmultext_features), 
('netmat', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_netmat_features), 
('netmat2', FunctionTransformer(square, validate=True), col_netmat_features)
 ],
remainder='drop')) , 
('win', FunctionTransformer(winsorize, validate=True)) , 
('e', Ridge(alpha=optpars['fnc_extended_ridge'][target]))
])) , 
('fnc_extended_ridgebag', Pipeline([ 
('columns', ColumnTransformer([('fnc', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_fnc_features), 
('internal_conn', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_internalconn_features), 
('external_conn', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_externalconn_features), 
('intmultext', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_intmultext_features), 
('netmat', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_netmat_features), 
('netmat2', FunctionTransformer(square, validate=True), col_netmat_features)
 ],
remainder='drop')) , 
('win', FunctionTransformer(winsorize, validate=True)) , 
('e', BaggingRegressor(Ridge(alpha=optpars['fnc_extended_ridgebag'][target]),
n_estimators=20,
random_state=42,
max_samples=0.8,
max_features=0.8))
])) , 
('fnc_simple_ridge', Pipeline([ 
('columns', ColumnTransformer([('internal_conn', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_internalconn_features), 
('external_conn', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_externalconn_features), 
('netmat', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_netmat_features)
 ],
remainder='drop')) , 
('win', FunctionTransformer(winsorize, validate=True)) , 
('poly', PolynomialFeatures()), 
('e', Ridge(alpha=optpars['fnc_simple_ridge'][target]))
])) , 
('spatial_maps_gmmax_wm_csf', Pipeline([ 
('columns', ColumnTransformer([('gm', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_gm_features), 
('gm2', FunctionTransformer(square, validate=True), col_gm_features), 
('wm', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_wm_features), 
('wm2', FunctionTransformer(square, validate=True), col_wm_features), 
('csf', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_csf_features), 
('csf2', FunctionTransformer(square, validate=True), col_csf_features)
 ],
remainder='drop')) , 
('win', FunctionTransformer(winsorize, validate=True)) , 
('e', Ridge(alpha=optpars['spatial_maps_gmmax_wm_csf'][target]))
])) , 
('sbm+funcdeweighted_win_svr',
Pipeline([('columns', ColumnTransformer([ 
('sbm', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_loading_features), 
('fnc', FunctionTransformer(down_scale, kw_args={'factor': 1/600}, validate=True), col_fnc_features), 
('internal_conn', FunctionTransformer(down_scale, kw_args={'factor': 1/600}, validate=True), col_internalconn_features), 
('external_conn', FunctionTransformer(down_scale, kw_args={'factor': 1/600}, validate=True), col_externalconn_features), 
('intmultext', FunctionTransformer(down_scale, kw_args={'factor': 1/600}, validate=True), col_intmultext_features)
 ],remainder='drop')) , 
('win', FunctionTransformer(winsorize, kw_args={'l': 0.0001, 'h': 0.0001}, validate=True)) , 
('e', SVR(C=optpars['sbm+funcdeweighted_win_svr'][target], cache_size=1000, gamma='scale'))
])) , 
('fnc_graph_ridge_alpha',
Pipeline([ 
('columns', ColumnTransformer([('sbm', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_loading_features), 
('fnc', FunctionTransformer(down_scale, kw_args={'factor': 1/600}, validate=True), col_fnc_features), 
('degree', FunctionTransformer(down_scale, kw_args={'factor': optpars['fnc_graph_ridge_degree_scale'][target]}, validate=True), col_degree_features), 
('clustering_r', FunctionTransformer(down_scale, kw_args={'factor': optpars['fnc_graph_ridge_clustering_r_scale'][target]}, validate=True), col_clustering_r_features), 
('clustering_i', FunctionTransformer(down_scale, kw_args={'factor': optpars['fnc_graph_ridge_clustering_i_scale'][target]}, validate=True), col_clustering_i_features), 
('betweenness', FunctionTransformer(down_scale, kw_args={'factor': optpars['fnc_graph_ridge_betweenness_scale'][target]}, validate=True), col_betweenness_centrality_features), 
('eigenvec', FunctionTransformer(down_scale, kw_args={'factor': optpars['fnc_graph_ridge_eigenvec_scale'][target]}, validate=True), col_eigenvector_centrality_features), 
('gm', FunctionTransformer(down_scale, kw_args={'factor': optpars['fnc_graph_ridge_gm_scale'][target]}, validate=True), col_gm_features), 
('csf', FunctionTransformer(down_scale, kw_args={'factor': optpars['fnc_graph_ridge_csf_scale'][target]}, validate=True), col_csf_features),
],
remainder='drop')) , 
('var', VarianceThreshold(0)) , 
('e', Ridge(alpha=optpars['fnc_graph_ridge_alpha'][target]))
])) , 
('gm_all_ridge',
Pipeline([ 
('columns', ColumnTransformer([('wm', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_wm_features), 
('wm2', FunctionTransformer(square, validate=True), col_wm_features), 
('csf', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_csf_features), 
('csf2', FunctionTransformer(square, validate=True), col_csf_features), 
('gm_all', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_gm_all)
 ],
remainder='drop')) , 
('win', FunctionTransformer(winsorize, validate=True)) , 
('e', Ridge(alpha=optpars['gm_all_ridge_alpha'][target]))
])) , 
('sbm_PNI_ridge',
Pipeline([('columns', ColumnTransformer([ 
('sbm', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_loading_features), 
('PNI', FunctionTransformer(down_scale, kw_args={'factor': 1}, validate=True), col_pni)
 ],remainder='drop')) , 
('e', Ridge(alpha=optpars['sbm_PNI_ridge_alpha'][target]))
])) ,
]
if target=='domain1_var1' or target=='domain2_var2':
estimators = estimators[2:]
print([e[0] for e in estimators])
 reg[target] = StackingRegressor(estimators=estimators, final_estimator=RidgeCV(alphas=[500, 1000, 5000, 10000]), passthrough=False, verbose=0, cv=6, n_jobs=1)
 reg[target].fit(dfnona[features], dfnona[target].values)
 print(metric(reg[target].predict(dfnona[features]), dfnona[target].values))
cv_score = cross_val_score(reg[target],
dfnona[features],
dfnona[target].values,
scoring=make_scorer(metric, greater_is_better=False))
print(np.mean(cv_score), cv_score)
 test_result[target] = reg[target].predict(test_result[features] )<save_to_csv>
dataset.LandSlope = dataset.LandSlope.apply(lambda x: 1 if x == 2 else 0 )
sub_df = pd.melt(test_result[["Id", "age", "domain1_var1", "domain1_var2", "domain2_var1", "domain2_var2"]], id_vars=["Id"], value_name="Predicted")
 sub_df["Id"] = sub_df["Id"].astype("str")+ "_" + sub_df["variable"].astype("str")
 sub_df = sub_df.drop("variable", axis=1 ).sort_values("Id")
 assert sub_df.shape[0] == test_df.shape[0]*5
sub_df.to_csv("submission.csv", index=False)
 sub_df.head(10 )<load_from_csv>
dataset.OverallQual = dataset.OverallQual.apply(lambda x: 2 if x == 1 else x )
ridge = pd.read_csv('/kaggle/input/trends-master-ensemble/submission_ridge.csv')
 modra = pd.read_csv('/kaggle/input/trends-master-ensemble/sub.csv')
 tunguz_n_IC20 = pd.read_csv('/kaggle/input/rapids-ensemblenoic20/submission_rapids_ensemble(1 ).csv')
 rapids = pd.read_csv('/kaggle/input/trends-master-ensemble/submission1.csv')
 bagging = pd.read_csv('/kaggle/input/baggingregressor-rapids-ensemble/submission_rapids_ensemble_with_baggingregressor.csv')
 SS_skewed = pd.read_csv('/kaggle/input/rapids-ensemblenoic20/06163.csv')
 stack = pd.read_csv('/kaggle/input/trends-multi-layer-model/submission.csv' )<create_dataframe>
dataset.OverallCond = dataset.OverallCond.apply(lambda x: 3 if x < 3 else x )
all_subs = pd.DataFrame(columns=['Id', 'modra', 'bagging', 'SS_skewed'])
 all_subs['Id'] = ridge['Id']
all_subs['modra'] = modra['Predicted']
all_subs['bagging'] = bagging['Predicted']
all_subs['SS_skewed'] = SS_skewed['Predicted']<merge>
dataset.ExterCond = dataset.ExterCond.apply(lambda x: 0 if x == 1 else x)
 dataset.ExterCond = dataset.ExterCond.apply(lambda x: 1 if x >= 2 else x )
all_subs['stack'] = all_subs.merge(stack, how='left', left_on='Id', right_on='Id')['Predicted']<define_variables>
dataset.BsmtCond = dataset.BsmtCond.apply(lambda x: 0 if x == 1 else x )
scores = {
'modra' : math.e **(5.0 - 0.33),
'bagging': math.e **(5.0 - 0.31),
'SS_skewed' : math.e **(5.0 - 0.1),
'stack' : math.e **(5.0 - 0.0)
 }<define_variables>
dataset.BsmtFinType1 = dataset.BsmtFinType1.astype(str )
columns = list(all_subs.columns)
 columns.remove('Id')
 total = 0
for col in columns:
total += scores[col]
total<feature_engineering>
dataset.BsmtFinType2 = dataset.BsmtFinType2.astype(str )
weight_sum = 0
for col in columns:
weight = scores[col] / total
weight_sum += weight
print(weight)
 all_subs[col] = all_subs[col] * weight
weight_sum<prepare_output>
dataset.HeatingQC = dataset.HeatingQC.apply(lambda x: 1 if x == 0 else x )
all_subs['Predicted'] = + all_subs['modra'] + all_subs['bagging'] + all_subs['SS_skewed'] + all_subs['stack']<prepare_output>
dataset.Electrical = dataset.Electrical.apply(lambda x: 2 if x < 2 else x )
sub = all_subs[['Id', 'Predicted']]
sub<save_to_csv>
dataset.Functional = dataset.Functional.apply(lambda x: 2 if x < 2 else x )
sub.to_csv('blend_sub.csv', index=False )<compute_test_metric>
dataset.Functional = dataset.Functional.astype(str )
def metric(y_true, y_pred):
return np.mean(np.sum(np.abs(y_true - y_pred), axis=0)/np.sum(y_true, axis=0))<load_from_csv>
dataset.GarageQual = dataset.GarageQual.apply(lambda x: 0 if x == 1 else x)
 dataset.GarageQual = dataset.GarageQual.apply(lambda x: 4 if x == 5 else x )
fnc_df = pd.read_csv(".. /input/trends-assessment-prediction/fnc.csv")
 loading_df = pd.read_csv(".. /input/trends-assessment-prediction/loading.csv")
 fnc_features, loading_features = list(fnc_df.columns[1:]), list(loading_df.columns[1:])
 df = fnc_df.merge(loading_df, on="Id")
 labels_df = pd.read_csv(".. /input/trends-assessment-prediction/train_scores.csv")
 labels_df['is_train'] = True
df = df.merge(labels_df, on="Id", how="left")
 test_df = df[df["is_train"] != True].copy()
df = df[df["is_train"] == True].copy()
df.shape, test_df.shape<prepare_x_and_y>
dataset.GarageCond = dataset.GarageCond.apply(lambda x: 0 if x == 1 else x)
 dataset.GarageCond = dataset.GarageCond.apply(lambda x: 3 if x > 3 else x )
for i in [500]:
tic = time()
FNC_SCALE = 1/i
df[fnc_features] *= FNC_SCALE
test_df[fnc_features] *= FNC_SCALE
NUM_FOLDS = 3
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 features = loading_features + fnc_features
overal_score = 0
for target, c, w in [("age", 100, 0.3),("domain1_var1", 10, 0.175),("domain1_var2", 10, 0.175),("domain2_var1", 10, 0.175),("domain2_var2", 10, 0.175)]:
y_oof = np.zeros(df.shape[0])
 y_test = np.zeros(( test_df.shape[0], NUM_FOLDS))
for f,(train_ind, val_ind)in enumerate(kf.split(df, df)) :
train_df, val_df = df.iloc[train_ind], df.iloc[val_ind]
train_df = train_df[train_df[target].notnull() ]
model = RFR(n_estimators=100)
 model.fit(train_df[features], train_df[target])
 s = pickle.dumps(model)
 joblib.dump(model, 'model'+str(i)+'.pkl')
 y_oof[val_ind] = model.predict(val_df[features])
 y_test[:, f] = model.predict(test_df[features])
 df["pred_{}".format(target)] = y_oof
test_df[target] = y_test.mean(axis=1)
 score = metric(df[df[target].notnull() ][target].values, df[df[target].notnull() ]["pred_{}".format(target)].values)
 overal_score += w*score
print(target, np.round(score, 4))
print()
print("Overall score:", np.round(overal_score, 4))
sub_df = pd.melt(test_df[["Id", "age", "domain1_var1", "domain1_var2", "domain2_var1", "domain2_var2"]], id_vars=["Id"], value_name="Predicted")
 sub_df["Id"] = sub_df["Id"].astype("str")+ "_" + sub_df["variable"].astype("str")
 sub_df = sub_df.drop("variable", axis=1 ).sort_values("Id")
 assert sub_df.shape[0] == test_df.shape[0]*5
sub_df.head(10)
 sub_df.to_csv("submission"+str(i)+".csv", index=False)
 toc = time()
print("Time taken for "+str(i)+" "+str(toc-tic))<install_modules>
dataset.GarageCond = dataset.GarageCond.astype(str )
!cp.. /input/rapids/rapids.0.13.0 /opt/conda/envs/rapids.tar.gz
!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz > /dev/null
sys.path = ["/opt/conda/envs/rapids/lib/python3.6/site-packages"] + sys.path
sys.path = ["/opt/conda/envs/rapids/lib/python3.6"] + sys.path
sys.path = ["/opt/conda/envs/rapids/lib"] + sys.path
!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/<compute_test_metric>
dataset = dataset.drop(columns='PoolQC' )
def metric(y_true, y_pred):
return np.mean(np.sum(np.abs(y_true - y_pred), axis=0)/np.sum(y_true, axis=0))<load_from_csv>
dataset.Fence = dataset.Fence.astype(str )
fnc_df = cudf.read_csv(".. /input/trends-assessment-prediction/fnc.csv")
 loading_df = cudf.read_csv(".. /input/trends-assessment-prediction/loading.csv")
 fnc_features, loading_features = list(fnc_df.columns[1:]), list(loading_df.columns[1:])
 df = fnc_df.merge(loading_df, on="Id")
 labels_df = cudf.read_csv(".. /input/trends-assessment-prediction/train_scores.csv")
 labels_df["is_train"] = True
df = df.merge(labels_df, on="Id", how="left")
 test_df = df[df["is_train"] != True].copy()
df = df[df["is_train"] == True].copy()<normalization>
cat_nominal = ['MSSubClass','MSZoning','Street','Alley','LandContour','LotConfig','Neighborhood',
'Condition1','Condition2','BldgType','HouseStyle','RoofStyle','RoofMatl','Exterior1st',
'Exterior2nd','MasVnrType','Foundation','Heating','GarageType','MiscFeature',
'SaleType','SaleCondition']
FNC_SCALE = 1/600
df[fnc_features] *= FNC_SCALE
test_df[fnc_features] *= FNC_SCALE<create_dataframe>
dataset.MSSubClass = dataset.MSSubClass.astype(str )
%%time
warnings.filterwarnings("ignore", message="Expected column")
 df_model1 = df.copy()
df_model2 = df.copy()
df_model3 = df.copy()
NUM_FOLDS = 7
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 features = loading_features + fnc_features
blend_weights = {"age": [0.4, 0.55, 0.05],
"domain1_var1": [0.55, 0.15, 0.3],
"domain1_var2": [0.45, 0.0, 0.55],
"domain2_var1": [0.55, 0.15, 0.3],
"domain2_var2": [0.5, 0.05, 0.45]}
overall_score = 0
for target, c, w in [("age", 60, 0.3),("domain1_var1", 12, 0.175),("domain1_var2", 8, 0.175),("domain2_var1", 9, 0.175),("domain2_var2", 12, 0.175)]:
y_oof = np.zeros(df.shape[0])
 y_oof_model_1 = np.zeros(df.shape[0])
 y_oof_model_2 = np.zeros(df.shape[0])
 y_oof_model_3 = np.zeros(df.shape[0])
 y_test = np.zeros(( test_df.shape[0], NUM_FOLDS))
for f,(train_ind, val_ind)in enumerate(kf.split(df, df)) :
train_df, val_df = df.iloc[train_ind], df.iloc[val_ind]
train_df = train_df[train_df[target].notnull() ]
model_1 = SVR(C=c, cache_size=3000.0)
 model_1.fit(train_df[features].values, train_df[target].values)
 model_2 = Ridge(alpha = 0.0001)
 model_2.fit(train_df[features].values, train_df[target].values)
 model_3 = BaggingRegressor(Ridge(alpha = 0.0001), n_estimators=30, random_state=42, max_samples=0.3, max_features=0.3)
 model_3.fit(train_df.to_pandas() [features].values, train_df.to_pandas() [target].values)
 val_pred_1 = model_1.predict(val_df[features])
 val_pred_2 = model_2.predict(val_df[features])
 val_pred_3 = model_3.predict(val_df.to_pandas() [features])
 val_pred_3 = cudf.from_pandas(pd.Series(val_pred_3))
test_pred_1 = model_1.predict(test_df[features])
 test_pred_2 = model_2.predict(test_df[features])
 test_pred_3 = model_3.predict(test_df.to_pandas() [features])
 test_pred_3 = cudf.from_pandas(pd.Series(test_pred_3))
val_pred = blend_weights[target][0]*val_pred_1+blend_weights[target][1]*val_pred_2+blend_weights[target][2]*val_pred_3
val_pred = cp.asnumpy(val_pred.values.flatten())
 test_pred = blend_weights[target][0]*test_pred_1+blend_weights[target][1]*test_pred_2+blend_weights[target][2]*test_pred_3
test_pred = cp.asnumpy(test_pred.values.flatten())
 y_oof[val_ind] = val_pred
y_oof_model_1[val_ind] = val_pred_1
y_oof_model_2[val_ind] = val_pred_2
y_oof_model_3[val_ind] = val_pred_3
y_test[:, f] = test_pred
df["pred_{}".format(target)] = y_oof
df_model1["pred_{}".format(target)] = y_oof_model_1
df_model2["pred_{}".format(target)] = y_oof_model_2
df_model3["pred_{}".format(target)] = y_oof_model_3
test_df[target] = y_test.mean(axis=1)
 score = metric(df[df[target].notnull() ][target].values, df[df[target].notnull() ]["pred_{}".format(target)].values)
 overall_score += w*score
score_model1 = metric(df_model1[df_model1[target].notnull() ][target].values, df_model1[df_model1[target].notnull() ]["pred_{}".format(target)].values)
 score_model2 = metric(df_model2[df_model2[target].notnull() ][target].values, df_model2[df_model1[target].notnull() ]["pred_{}".format(target)].values)
 score_model3 = metric(df_model3[df_model3[target].notnull() ][target].values, df_model3[df_model1[target].notnull() ]["pred_{}".format(target)].values)
 print(f"For {target}:")
 print("SVR:", np.round(score_model1, 6))
print("Ridge:", np.round(score_model2, 6))
print("BaggingRegressor:", np.round(score_model3, 6))
print("Ensemble:", np.round(score, 6))
print()
print("Overall score:", np.round(overall_score, 6))<data_type_conversions>
dataset = dataset.drop(columns='Street' )
sub_df = cudf.melt(test_df[["Id", "age", "domain1_var1", "domain1_var2", "domain2_var1", "domain2_var2"]], id_vars=["Id"], value_name="Predicted")
 sub_df["Id"] = sub_df["Id"].astype("str")+ "_" + sub_df["variable"].astype("str")
 sub_df = sub_df.drop("variable", axis=1 ).sort_values("Id")
 assert sub_df.shape[0] == test_df.shape[0]*5
sub_df.head(10 )<save_to_csv>
dataset.LotConfig = dataset.LotConfig.apply(lambda x: 'FR2' if x == 'FR3' else x )
sub_df.to_csv("submission_rapids_ensemble_with_baggingregressor.csv", index=False )<import_modules>
dataset.Condition1 = dataset.Condition1.apply(lambda x: 'RRs' if(x == 'RRNe' or x == 'RRAe' or x == 'RRNn' or x == 'RRAn')else x )
import numpy as np
import pandas as pd
from sklearn.multioutput import MultiOutputRegressor
from sklearn.linear_model import BayesianRidge
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import KFold<define_variables>
dataset = dataset.drop(columns='Condition2' )
DATA_PATH = '/kaggle/input/trends-assessment-prediction/'<load_from_csv>
dataset.RoofStyle = dataset.RoofStyle.apply(lambda x: 'Other' if(x == 'Gambrel' or x == 'Mansard' or x == 'Flat' or x == 'Shed')else x )
loading = pd.read_csv(DATA_PATH + 'loading.csv')
 loading.head()<load_from_csv>
dataset = dataset.drop(columns='RoofMatl' )
fnc = pd.read_csv(DATA_PATH + 'fnc.csv')
 fnc.head()<merge>
dataset.Exterior1st = dataset.Exterior1st.apply(lambda x: 'Other' if(x == 'BrkComm' or x == 'AsphShn' or x == 'Stone' or x == 'ImStucc' or x== 'CBlock')else x )
dataset = loading.merge(fnc, on='Id')
 dataset.head()<load_from_csv>
dataset = dataset.drop(columns = 'Exterior2nd' )
y_train = pd.read_csv(DATA_PATH + 'train_scores.csv')
 print('Number of training samples: {}'.format(len(y_train)))
 y_train.head()<categorify>
dataset.MiscFeature = dataset.MiscFeature.apply(lambda x: 0 if x == 'NA' else 1 )
y_train = y_train.fillna(y_train.mean())
 y_train.head()<filter>
dataset.SaleType = dataset.SaleType.apply(lambda x: x if(x == 'WD' or x == 'New')else 'Other' )
x_train = dataset.loc[dataset['Id'].isin(y_train['Id'])]
x_train.head()<define_variables>
dataset.SaleCondition = dataset.SaleCondition.apply(lambda x: x if(x == 'Normal' or x == 'Abnormal' or x == 'Partial')else 'Other' )
x_test = dataset.loc[~dataset['Id'].isin(y_train['Id'])]
test_ids = x_test['Id']
print('Number of test samples: {}'.format(len(x_test)))
 x_test.head()<prepare_x_and_y>
X = dataset.loc[:,dataset.columns.difference(['SalePrice', 'Id'])]
Y = dataset.SalePrice
X = pd.get_dummies(X )
x_train = x_train.drop('Id', axis=1 ).values
x_test = x_test.drop('Id', axis=1 ).values
y_train = y_train.drop('Id', axis=1 ).values<choose_model_class>
scaler = RobustScaler()
X = pd.DataFrame(scaler.fit_transform(X))
model = MultiOutputRegressor(BayesianRidge(normalize=True), n_jobs=-1 )<compute_test_metric>
X_train = X.iloc[0:1457].as_matrix()
Y_train = Y.iloc[0:1457].as_matrix()
X_test = X.iloc[1457:].as_matrix()
test_ids = dataset.Id.iloc[1457:]
def score(y_pred, y_true):
return sum(list(map(lambda w, s: w * s, [.3,.175,.175,.175,.175], np.sum(np.abs(y_true - y_pred), axis=0)/np.sum(y_true, axis=0))))<train_model>
lasso = Lasso(alpha= 0.0005 )
n = 7
y_test = np.zeros(( len(x_test), 5, n))
scores = np.zeros(n)
 for i,(train_indexes, valid_indexes)in enumerate(KFold(n, shuffle=True, random_state=0 ).split(x_train)) :
print('Fold {} of {}...'.format(i + 1, n))
x_train_f, x_valid = x_train[train_indexes], x_train[valid_indexes]
y_train_f, y_valid = y_train[train_indexes], y_train[valid_indexes]
model.fit(x_train_f, y_train_f)
 y_pred = model.predict(x_valid)
 scores[i] = score(y_pred, y_valid)
 print('Score = {}'.format(scores[i]))
y_test[:,:,i] = model.predict(x_test )<compute_test_metric>
elastic = ElasticNet(alpha=0.0005, l1_ratio=.9 )
print('Average score = {}'.format(scores.mean()))<feature_engineering>
y_test = y_test.mean(axis=2)
 print(y_test )<prepare_output>
k_ridge = KernelRidge(alpha=0.1, coef0=2.5, degree=3, gamma=None, kernel='polynomial',
kernel_params=None )
outputs = ['age', 'domain1_var1', 'domain1_var2', 'domain2_var1', 'domain2_var2']
ids = ['{}_{}'.format(int(id_), output)for id_ in test_ids for output in outputs]
predicted = y_test.reshape(5 * len(y_test))
assert len(predicted)== 29385
submission = pd.DataFrame({'Id': ids, 'Predicted': predicted})
 submission.head(10 )<save_to_csv>
submission.to_csv('submission.csv', index = False)
 !head submission.csv<set_options>
g_boost = GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse',
init=None, learning_rate=0.01, loss='huber',
max_depth=3, max_features=18, max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=3, min_samples_split=5,
min_weight_fraction_leaf=0.0, n_estimators=3300,
n_iter_no_change=None,random_state=None, subsample=1.0, tol=0.0001,
validation_fraction=0.1, verbose=0, warm_start=False )
%matplotlib inline<load_from_csv>
df_train=pd.read_csv('.. /input/train.csv')
 df_test=pd.read_csv('.. /input/test.csv' )<count_missing_values>
xg_boost = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=4, min_child_weight=1, missing=None, n_estimators=3300,
n_jobs=1, nthread=None, objective='reg:linear', random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
silent=None, subsample=0.3, verbosity=1 )
def check_missing_data(df):
total=df.isnull().sum().sort_values(ascending=False)
 percent=(( df.isnull().sum() /df.isnull().count())*100 ).sort_values(ascending=False)
 return pd.concat([total,percent],axis=1,keys=['Total','Percent'] )<prepare_output>
df_tmp=pd.DataFrame(df_train.nunique().sort_values() ,columns=['num_unique_values'] ).reset_index().rename(columns={'index':'Column_name'})
 df_tmp.head()<count_unique_values>
lgbm = LGBMRegressor(bagging_fraction=0.6, bagging_freq=9, bagging_seed=14,
boosting_type='gbdt', class_weight=None, colsample_bytree=0,
feature_fraction=0.1, feature_fraction_seed=1,
importance_type='split', learning_rate=0.01, max_bin=65,
max_depth=4, min_child_samples=20, min_child_weight=0.001,
min_data_in_leaf=5, min_split_gain=0.0, min_sum_hessian_in_leaf=5,
n_estimators=3300, n_jobs=-1, num_leaves=7,
objective='regression', random_state=None, reg_alpha=0.2,
reg_lambda=0.1, silent=True, subsample=1.0,
subsample_for_bin=200000, subsample_freq=0 )
def col_name_with_n_unique_value(df,n):
df1=pd.DataFrame(df.nunique().sort_values() ,columns=['num_unique_values'] ).reset_index()
col_name=list(df1[df1.num_unique_values==1]['index'])
 print('number of columns with only',n,'unique values are: ',len(col_name))
return col_name<count_unique_values>
stacking = StackingRegressor(regressors=(elastic, g_boost, k_ridge),
meta_regressor = lasso)
 param_grid = {}
stack = GridSearchCV(stacking,
param_grid = param_grid,
cv = 10,
scoring = "neg_mean_squared_error",
n_jobs = 5,
verbose = 1)
 stack.fit(X_train,Y_train)
 s_best = stack.best_estimator_
stack.best_score_
col_to_drop=col_name_with_n_unique_value(df_train,1 )<drop_column>
stacking = s_best
del df_train,df_test,df_tmp
gc.collect()<normalization>
voting = VotingRegressor(estimators=[('xgboost', xg_boost), 
('lgbm', lgbm), 
('stacking', stacking)])
 v_param_grid = {}
gsV = GridSearchCV(voting,
param_grid = v_param_grid,
cv = 10,
scoring = "neg_mean_squared_error",
n_jobs = 5,
verbose = 1)
 gsV.fit(X_train,Y_train)
 v_best = gsV.best_estimator_
gsV.best_score_
sc = MinMaxScaler(feature_range=(0,1))
train_sc = sc.fit_transform(train)
 test_sc = sc.transform(test )<split>
voting = v_best
X_train, X_val, y_train, y_val = train_test_split(train_sc, t, test_size=0.2, random_state=0)
 print(X_train.shape, y_train.shape)
 print(X_val.shape, y_val.shape)
 print(test_sc.shape, test_sc.shape )<set_options>
y_vote = np.expm1(voting.predict(X_test))
lasso.fit(X_train,Y_train)
 y_lasso = np.expm1(lasso.predict(X_test))
del train_sc,train,test
gc.collect()<create_dataframe>
y_pred =(0.6*y_vote + 0.4*y_lasso )
<create_dataframe><EOS> 
submission = pd.DataFrame(test_ids.values, columns = ["Id"])
 submission['SalePrice'] = list(y_pred)
 submission.to_csv('submission.csv',index=False )