Kaggle competition: House Prices - Advanced Regression Techniques
train_scores = pd.read_csv('/kaggle/input/trends-assessment-prediction/train_scores.csv' ).sort_values(by='Id')
 loadings = pd.read_csv('/kaggle/input/trends-assessment-prediction/loading.csv')
 fnc = pd.read_csv('/kaggle/input/trends-assessment-prediction/fnc.csv')
 sample = pd.read_csv('/kaggle/input/trends-assessment-prediction/sample_submission.csv')
 reveal = pd.read_csv('/kaggle/input/trends-assessment-prediction/reveal_ID_site2.csv')
 icn_nums = pd.read_csv('/kaggle/input/trends-assessment-prediction/ICN_numbers.csv')
<import_modules>
df_train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
import re
from tqdm import tqdm<feature_engineering>
r = re.compile('\d+')
 col_dict = {}
for col in fnc.columns:
ind = r.findall(col)
 if ind:
col_dict[col] = [int(i)for i in ind]<prepare_x_and_y>
def get_matrix(df_row, return_idx=False):
matrix = np.zeros(( 100, 100))
for col in df_row.index[1:]:
i, j = col_dict[col]
matrix[i, j] = df_row[col]
matrix += matrix.T
idx = np.array([ 2, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 16, 17, 18, 20, 21, 23,
27, 32, 33, 37, 38, 40, 43, 45, 48, 51, 53, 54, 55, 56, 61, 62, 63,
66, 67, 68, 69, 70, 71, 72, 77, 79, 80, 81, 83, 84, 88, 93, 94, 96,
98, 99])
 if return_idx:
return matrix[:, idx][idx, :], idx
return matrix[:, idx][idx, :]<create_dataframe>
df_train['TotalSF'] = df_train['TotalBsmtSF']+df_train['1stFlrSF']+df_train['2ndFlrSF']
df_train=df_train.drop(columns={'1stFlrSF', '2ndFlrSF','TotalBsmtSF'})
 df_train['wholeExterior'] = df_train['Exterior1st']+df_train['Exterior2nd']
df_train=df_train.drop(columns={'Exterior1st','Exterior2nd'})
 df_train['Bsmt'] = df_train['BsmtFinSF1']+ df_train['BsmtFinSF2']
df_train = df_train.drop(columns={'BsmtFinSF1','BsmtFinSF2'})
 df_train['TotalBathroom'] = df_train['FullBath'] + df_train['HalfBath']
df_train = df_train.drop(columns={'FullBath','HalfBath'})
 df_test['TotalSF'] = df_test['TotalBsmtSF']+df_test['1stFlrSF']+df_test['2ndFlrSF']
df_test=df_test.drop(columns={'1stFlrSF', '2ndFlrSF','TotalBsmtSF'})
 df_test['wholeExterior'] = df_test['Exterior1st']+df_test['Exterior2nd']
df_test=df_test.drop(columns={'Exterior1st','Exterior2nd'})
 df_test['Bsmt'] = df_test['BsmtFinSF1']+ df_test['BsmtFinSF2']
df_test = df_test.drop(columns={'BsmtFinSF1','BsmtFinSF2'})
 df_test['TotalBathroom'] = df_test['FullBath'] + df_test['HalfBath']
df_test = df_test.drop(columns={'FullBath','HalfBath'} )
_, idx = get_matrix(fnc.iloc[0], return_idx=True)
 degrees = pd.DataFrame(degrees, columns=idx)
 degrees['Id'] = fnc['Id']<define_variables>
frames = [df_train,df_test]
df = pd.concat(frames,keys=['train','test'] )
len(glob('/kaggle/input/trends-assessment-prediction/fMRI_train/*.mat')) , len(glob('/kaggle/input/trends-assessment-prediction/fMRI_test/*.mat'))<import_modules>
df_missing=df.isnull().sum().sort_values(ascending=False )
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import Ridge, Lasso
from sklearn.svm import SVR
from sklearn.model_selection import GridSearchCV, KFold, cross_val_predict<merge>
cat_col = df.select_dtypes(include=['object'])
 cat_col.isnull().sum()
cat_col.columns
train_ids = sorted(loadings[loadings['Id'].isin(train_scores.Id)]['Id'].values)
 test_ids = sorted(loadings[~loadings['Id'].isin(train_scores.Id)]['Id'].values)
 test_prediction = pd.DataFrame(test_ids, columns=['Id'], dtype=str)
 target_columns =('age', 'domain1_var1', 'domain1_var2','domain2_var1','domain2_var2')
 fnc_columns = fnc.columns[1:]
degrees_columns = degrees.columns[:-1]
data = pd.merge(loadings, train_scores, on='Id')
 data = pd.merge(data, fnc, on='Id')
 data = pd.merge(data, degrees, on='Id')
 X_test = pd.merge(loadings[loadings.Id.isin(test_ids)], fnc, on='Id')
 X_test = pd.merge(X_test, degrees, on='Id' ).drop('Id', axis=1 )<compute_test_metric>
df['LotFrontage'] = df['LotFrontage'].fillna(value=0)
 df['GarageYrBlt'] = df['GarageYrBlt'].fillna(value=0)
 df['MasVnrArea'] = df['MasVnrArea'].fillna(value=0)
 df['BsmtFullBath'] = df['BsmtFullBath'].fillna(value=0)
 df['BsmtHalfBath'] = df['BsmtHalfBath'].fillna(value=0)
 df['GarageArea'] = df['GarageArea'].fillna(value=0)
 df['GarageCars'] = df['GarageCars'].fillna(value=0)
 df['BsmtUnfSF'] = df['BsmtUnfSF'].fillna(value=0)
 df['Bsmt'] = df['Bsmt'].fillna(value=0)
 df['TotalSF'] = df['TotalSF'].fillna(value=0 )
def MAPE(y_true, y_pred, **kwargs):

 return np.sum(np.abs(y_true - y_pred)) / y_true.sum()
mape_scorer = make_scorer(MAPE, greater_is_better=False )<find_best_model_class>
df['MSZoning'] = df['MSZoning'].fillna(value='None')
 df['GarageQual'] = df['GarageQual'].fillna(value='None')
 df['GarageCond'] = df['GarageCond'].fillna(value='None')
 df['GarageFinish'] = df['GarageFinish'].fillna(value='None')
 df['GarageType'] = df['GarageType'].fillna(value='None')
 df['BsmtExposure'] = df['BsmtExposure'].fillna(value='None')
 df['BsmtCond'] = df['BsmtCond'].fillna(value='None')
 df['BsmtQual'] = df['BsmtQual'].fillna(value='None')
 df['BsmtFinType2'] = df['BsmtFinType2'].fillna(value='None')
 df['BsmtFinType1'] = df['BsmtFinType1'].fillna(value='None')
 df['MasVnrType'] = df['MasVnrType'].fillna(value='None')
 df['Utilities'] = df['Utilities'].fillna(value='None')
 df['Functional'] = df['Functional'].fillna(value='None')
 df['Electrical'] = df['Electrical'].fillna(value='None')
 df['KitchenQual'] = df['KitchenQual'].fillna(value='None')
 df['SaleType'] = df['SaleType'].fillna(value='None')
 df['wholeExterior'] = df['wholeExterior'].fillna(value='None' )
best_models = {}
total_score = []
for col in target_columns:
X_train = data.dropna(subset=[col], axis=0 ).drop(list(target_columns), axis=1 ).drop('Id', axis=1)
 X_train[fnc_columns] /= 500
y_train = data.dropna(subset=[col], axis=0)[col]
gs.fit(X_train, y_train)
 best_models[col] = gs.best_estimator_
y_pred = cross_val_predict(gs.best_estimator_, X_train, y_train, cv=cv, n_jobs=-1)
 total_score.append(MAPE(y_train, y_pred))
print(col, MAPE(y_train, y_pred))
total_score = np.array(total_score)
 print(f'Total score: {np.sum(total_score*[.3,.175,.175,.175,.175])}' )<predict_on_test>
df = df.drop(columns={'PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu'} )
def get_pred(col, model):
X_train = data.dropna(subset=[col], axis=0 ).drop(list(target_columns), axis=1 ).drop('Id', axis=1)
 X_train[fnc_columns] /= 500
y_train = data.dropna(subset=[col], axis=0)[col]
y_pred = cross_val_predict(model, X_train, y_train, cv=cv, n_jobs=-1)
 return y_pred<predict_on_test>
df_main = pd.get_dummies(df)
 df_main
X_test[fnc_columns] /= 500
for col in target_columns:
test_prediction[col] = best_models[col].predict(X_test )<compute_test_metric>
EID = df_main.loc['test']
df_test = df_main.loc['test']
df_train = df_main.loc['train']
EID = EID.Id
<categorify>
df_test.drop(['SalePrice','Id'], axis =1, inplace=True )
def make_sub(test_prediction):

 target_columns =('age', 'domain1_var1', 'domain1_var2','domain2_var1','domain2_var2')
 _columns =(0,1,2,3,4)
 tst = test_prediction.rename(columns=dict(zip(target_columns, _columns)))
 tst = tst.melt(id_vars='Id',
value_vars=_columns,
value_name='Predicted')
 tst['target_type'] = tst.variable.map(dict(zip(_columns, target_columns)))
 tst['Id_'] = tst[['Id', 'target_type']].apply(lambda x: '_'.join(( str(x[0]), str(x[1]))), axis=1)
 return tst.sort_values(by=['Id', 'variable'])\ 
.drop(['Id', 'variable', 'target_type'],axis=1)\ 
.rename(columns={'Id_':'Id'})\ 
.reset_index(drop=True)\
[['Id', 'Predicted']]<prepare_output>
X_train = df_train.drop(['SalePrice','Id'], axis = 1)
 y_train = df_train['SalePrice']
sub = make_sub(test_prediction)
 sub.head()<save_to_csv>
xgboost = xgboost.XGBRegressor(learning_rate=0.05,
colsample_bytree = 0.5,
subsample = 0.8,
n_estimators=1000,
max_depth=5,
gamma=5)
 xgboost.fit(X_train, y_train)
 y_pred = xgboost.predict(df_test )
sub.to_csv('ridge_mape_500.csv', index=False )<install_modules>
main_submission = pd.DataFrame({'Id': EID, 'SalePrice': y_pred})
 main_submission.to_csv("submission.csv", index=False)
 main_submission.head()