Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<drop_column>
import pandas as pd
from tqdm import tqdm
numcols = train.drop('target',axis=1 ).select_dtypes(include='number' ).columns.values<concatenate>
train = pd.read_csv('.. /input/ames-housing-dataset/AmesHousing.csv')
 train.drop(['PID'], axis=1, inplace=True)
 origin = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 train.columns = origin.columns
test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 print('Train:{} Test:{}'.format(train.shape,test.shape))
all_data = train.append(test, sort=False ).reset_index(drop=True)
 del train, test
gc.collect()
all_data.head()<count_unique_values>
missing = test.isnull().sum()
missing = missing[missing>0]
train.drop(missing.index, axis=1, inplace=True)
 train.drop(['Electrical'], axis=1, inplace=True)
 test.dropna(axis=1, inplace=True)
 test.drop(['Electrical'], axis=1, inplace=True )
constant_column = [col for col in all_data.columns if all_data[col].nunique() == 1]
print('drop columns:', constant_column)
 all_data.drop(constant_column, axis=1, inplace=True )<data_type_conversions>
l_test = tqdm(range(0, len(test)) , desc='Matching')
 for i in l_test:
for j in range(0, len(train)) :
for k in range(1, len(test.columns)) :
if test.iloc[i,k] == train.iloc[j,k]:
continue
else:
break
else:
submission.iloc[i, 1] = train.iloc[j, -1]
break
l_test.close()
corr_matrix = all_data.corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [c for c in upper.columns if any(upper[c] > 0.95)]
del upper
drop_column = all_data.columns[to_drop]
print('drop columns:', drop_column)
<categorify>
submission.to_csv('submission.csv', index=False )
cols = [col for col in all_data.columns if col not in ['ID_code']]
for i, t in all_data.loc[:, cols].dtypes.iteritems() :
if t == object:
print(i)
 all_data[i] = pd.factorize(all_data[i])[0]<normalization>
submission.to_csv('submission.csv', index=False )