Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<init_hyperparams>
print(os.listdir(".. /input"))
params = {}
params["objective"] = "binary:logistic"
params["eval_metric"] = "logloss"
params["eta"] = 0.04
params["gamma"] = 0
params["scale_pos_weight"] = 0.3632
params["tree_method"] = "gpu_hist"
params["max_depth"] = 6
params["min_child_weight"] = 4
params["max_bin"] = 256
params["subsample"] = 0.8
params["colsample_bytree"] = 0.9
model4 = xgb.cv(params, dtrain, num_boost_round = 6000, nfold = 10,
feval = weighted_log_loss, early_stopping_rounds = 150,
verbose_eval = 50 )<init_hyperparams>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 test2 = pd.read_csv(".. /input/test.csv")
 houses = pd.concat([train,test],sort=False)
 print(train.shape)
 print(test.shape )
params = {}
params["objective"] = "binary:logistic"
params["eval_metric"] = "logloss"
params["eta"] = 0.02
params["gamma"] = 0
params["scale_pos_weight"] = 0.3632
params["tree_method"] = "gpu_hist"
params["max_depth"] = 6
params["min_child_weight"] = 4
params["max_bin"] = 256
params["subsample"] = 0.8
params["colsample_bytree"] = 0.9
model2 = xgb.cv(params, dtrain, num_boost_round = 6000, nfold = 10,
feval = weighted_log_loss, early_stopping_rounds = 150,
verbose_eval = 50 )<load_from_csv>
train.select_dtypes(include='object' ).isnull().sum()
params = {}
params["objective"] = "binary:logistic"
params["eval_metric"] = "logloss"
params["eta"] = 0.02
params["gamma"] = 0
params["scale_pos_weight"] = 0.3632
params["tree_method"] = "gpu_hist"
params["max_depth"] = 6
params["min_child_weight"] = 4
params["max_bin"] = 256
params["subsample"] = 0.8
params["colsample_bytree"] = 0.9
dtest = xgb.DMatrix(test)
 t = pd.read_csv(".. /input/quora-question-pairs/test.csv" )<save_to_csv>
train.select_dtypes(exclude='object' ).isnull().sum()
model = xgb.train(params, dtrain, num_boost_round = 3600)
 prediction = model.predict(dtest)
 sub = pd.DataFrame()
sub['test_id'] = t["test_id"]
sub['is_duplicate'] = prediction
sub.to_csv('submission3600.csv', index=False )<save_to_csv>
train.drop(['Alley', 'PoolQC','Fence', 'MiscFeature'], axis=1, inplace=True)
 test.drop(['Alley', 'PoolQC','Fence', 'MiscFeature'], axis=1, inplace=True)
model = xgb.train(params, dtrain, num_boost_round = 3800)
 prediction = model.predict(dtest)
 sub = pd.DataFrame()
sub['test_id'] = t["test_id"]
sub['is_duplicate'] = prediction
sub.to_csv('submission3800.csv', index=False )<save_to_csv>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure','BsmtFinType1',
'BsmtFinType2', 'FireplaceQu', 'GarageType', 'GarageFinish',
'GarageQual', 'GarageCond', 'PavedDrive', 'Electrical','MasVnrType'):
train[col]=train[col].fillna('Unknown')
 test[col]=test[col].fillna('Unknown')
model = xgb.train(params, dtrain, num_boost_round = 4100)
 prediction = model.predict(dtest)
 sub = pd.DataFrame()
sub['test_id'] = t["test_id"]
sub['is_duplicate'] = prediction
sub.to_csv('submission4100.csv', index=False )<load_from_csv>
print(train.GarageType.isnull().sum())
 print(test.GarageType.isnull().sum() )
df_train = pd.read_csv('.. /input/train.csv')
 df_train.head()<save_to_csv>
train['LotFrontage']=train['LotFrontage'].fillna(train.LotFrontage.mean())
 test['LotFrontage']= test['LotFrontage'].fillna(test.LotFrontage.mean())
 train['MasVnrArea']= train['MasVnrArea'].fillna(train.MasVnrArea.mean())
 test['MasVnrArea']= test['MasVnrArea'].fillna(test.MasVnrArea.mean())
 train['GarageYrBlt']= train['GarageYrBlt'].fillna(0)
 test['GarageYrBlt']= test['GarageYrBlt'].fillna(0)
 test['BsmtFinSF1']= test['BsmtFinSF1'].fillna(test.BsmtFinSF1.mean())
 test['BsmtFinSF2']= test['BsmtFinSF2'].fillna(test.BsmtFinSF2.mean())
 test['BsmtUnfSF']= test['BsmtUnfSF'].fillna(test.BsmtUnfSF.mean())
 test['BsmtFullBath']= test['BsmtFullBath'].fillna(test.BsmtFullBath.mean())
 test['BsmtHalfBath']= test['BsmtHalfBath'].fillna(test.BsmtHalfBath.mean())
 test['GarageCars']= test['GarageCars'].fillna(test.GarageCars.mean())
 test['TotalBsmtSF']= test['TotalBsmtSF'].fillna(test.TotalBsmtSF.mean())
 test['GarageArea'] = test['GarageArea'].fillna(test.GarageArea.mean())
p = df_train['is_duplicate'].mean()
print('Predicted score:', log_loss(df_train['is_duplicate'], np.zeros_like(df_train['is_duplicate'])+ p))
df_test = pd.read_csv('.. /input/test.csv')
 sub = pd.DataFrame({'test_id': df_test['test_id'], 'is_duplicate': p})
 sub.to_csv('naive_submission.csv', index=False)
 sub.head()<load_from_csv>
test.columns[test.isnull().any() ].tolist()
df_test = pd.read_csv('.. /input/test.csv')
 df_test.head()<define_variables>
train = train[train['GrLivArea']<4000]
print('Total number of question pairs for testing: {}'.format(len(df_test)) )<feature_engineering>
features = ['MSSubClass', 'LotFrontage', 'LotArea', 'OverallQual',
'OverallCond', 'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1',
'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF',
'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath',
'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd',
'Fireplaces', 'GarageYrBlt', 'GarageCars', 'GarageArea', 'WoodDeckSF',
'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea',
'MiscVal', 'MoSold', 'YrSold']
x_train, x_test, y_train, y_test = train_test_split(train[features],train['SalePrice'], train_size = 0.8,
test_size = 0.2, random_state=3)
 lm = LinearRegression()
lm.fit(x_train, y_train)
 print(lm.score(x_train, y_train))
print(lm.score(x_test, y_test))
qmarks = np.mean(train_qs.apply(lambda x: '?' in x))
math = np.mean(train_qs.apply(lambda x: '[math]' in x))
fullstop = np.mean(train_qs.apply(lambda x: '.' in x))
capital_first = np.mean(train_qs.apply(lambda x: x[0].isupper()))
capitals = np.mean(train_qs.apply(lambda x: max([y.isupper() for y in x])))
 numbers = np.mean(train_qs.apply(lambda x: max([y.isdigit() for y in x])))
 print('Questions with question marks: {:.2f}%'.format(qmarks * 100))
print('Questions with [math] tags: {:.2f}%'.format(math * 100))
print('Questions with full stops: {:.2f}%'.format(fullstop * 100))
print('Questions with capitalised first letters: {:.2f}%'.format(capital_first * 100))
print('Questions with capital letters: {:.2f}%'.format(capitals * 100))
print('Questions with numbers: {:.2f}%'.format(numbers * 100))<string_transform>
most_corr_features = ['OverallQual', 'YearBuilt', 'YearRemodAdd', 'MasVnrArea',
'BsmtFinSF1', 'TotalBsmtSF', 'GrLivArea',
'FullBath', 'Fireplaces', 'GarageCars']
steps = [ 
('scalar', StandardScaler()), 
('poly', PolynomialFeatures(degree=3)) , 
('model', Lasso(alpha=1000, fit_intercept=True))
]
pipeline = Pipeline(steps)
 x_train2, x_test2, y_train2, y_test2 = train_test_split(train[most_corr_features], train["SalePrice"], train_size=0.8, test_size=0.2, random_state=3)
 pipeline.fit(x_train2, y_train2)
 print(pipeline.score(x_train2, y_train2))
print(pipeline.score(x_test2, y_test2))
stops = set(stopwords.words("english"))
def word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
R =(len(shared_words_in_q1)+ len(shared_words_in_q2)) /(len(q1words)+ len(q2words))
return R
plt.figure(figsize=(15, 5))
train_word_match = df_train.apply(word_match_share, axis=1, raw=True)
 plt.hist(train_word_match[df_train['is_duplicate'] == 0], bins=20, normed=True, label='Not Duplicate')
 plt.hist(train_word_match[df_train['is_duplicate'] == 1], bins=20, normed=True, alpha=0.7, label='Duplicate')
 plt.legend()
plt.title('Label distribution over word_match_share', fontsize=15)
 plt.xlabel('word_match_share', fontsize=15 )<init_hyperparams>
len_train=train.shape[0]
houses=pd.concat([train,test], sort=False)
 for col in('MSZoning', 'Utilities', 'Exterior1st', 'Exterior2nd', 'KitchenQual', 'Functional', 'SaleType'):
houses[col]=houses[col].fillna('Unknown')
def get_weight(count, eps=10000, min_count=2):
if count < min_count:
return 0
else:
return 1 /(count + eps)
 eps = 5000
words =(" ".join(train_qs)).lower().split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }<sort_values>
houses["MSSubClass"] = houses["MSSubClass"].apply(str)
 houses["MoSold"] = houses["MoSold"].apply(str)
print('Most common words and weights:
')
 print(sorted(weights.items() , key=lambda x: x[1] if x[1] > 0 else 9999)[:10])
 print('
Least common words and weights: ')
(sorted(weights.items() , key=lambda x: x[1], reverse=True)[:10] )<feature_engineering>
houses["YrSold"] = houses["YrSold"].apply(str)
def tfidf_word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_weights = [weights.get(w, 0)for w in q1words.keys() if w in q2words] + [weights.get(w, 0)for w in q2words.keys() if w in q1words]
total_weights = [weights.get(w, 0)for w in q1words] + [weights.get(w, 0)for w in q2words]
R = np.sum(shared_weights)/ np.sum(total_weights)
 return R<compute_test_metric>
houses.drop("GarageArea", axis=1, inplace=True)
 houses=pd.get_dummies(houses)
 train=houses[:len_train]
test=houses[len_train:]
train.drop('Id', axis=1, inplace=True)
 test.drop('Id', axis=1, inplace=True)
 train['SalePrice']=np.log(train['SalePrice'])*100
x=train.drop('SalePrice', axis=1)
 y=train['SalePrice']
test=test.drop('SalePrice', axis=1 )
print('Original AUC:', roc_auc_score(df_train['is_duplicate'], train_word_match))
print(' TFIDF AUC:', roc_auc_score(df_train['is_duplicate'], tfidf_train_word_match.fillna(0)) )<prepare_x_and_y>
steps = [ 
('poly', PolynomialFeatures(degree=1)) , 
('model', Lasso(alpha=0.15, fit_intercept=True))
]
pipeline = Pipeline(steps)
 x_train2, x_test2, y_train2, y_test2 = train_test_split(x, y, train_size=0.8, test_size=0.2, random_state=3)
 pipeline.fit(x_train2, y_train2)
 print(rmse(actual=y_test2/100, predicted=pipeline.predict(x_test2)/100))
x_train = pd.DataFrame()
x_test = pd.DataFrame()
x_train['word_match'] = train_word_match
x_train['tfidf_word_match'] = tfidf_train_word_match
x_test['word_match'] = df_test.apply(word_match_share, axis=1, raw=True)
 x_test['tfidf_word_match'] = df_test.apply(tfidf_word_match_share, axis=1, raw=True)
 y_train = df_train['is_duplicate'].values<concatenate>
predictions1 = pipeline.predict(test )
pos_train = x_train[y_train == 1]
neg_train = x_train[y_train == 0]
p = 0.165
scale =(( len(pos_train)/(len(pos_train)+ len(neg_train)))/ p)- 1
while scale > 1:
neg_train = pd.concat([neg_train, neg_train])
 scale -=1
neg_train = pd.concat([neg_train, neg_train[:int(scale * len(neg_train)) ]])
 print(len(pos_train)/(len(pos_train)+ len(neg_train)))
 x_train = pd.concat([pos_train, neg_train])
 y_train =(np.zeros(len(pos_train)) + 1 ).tolist() + np.zeros(len(neg_train)).tolist()
del pos_train, neg_train<split>
params = {
'n_estimators': 300,
'max_depth': 3,
'learning_rate': 0.1,
'criterion': 'mae',
'min_impurity_decrease': 0.005,
'loss': 'huber',
'alpha': 0.99
}
gradient_boosting_regressor = GradientBoostingRegressor(**params)
 gradient_boosting_regressor.fit(x_train2, y_train2)
 print(rmse(actual=y_test2/100, predicted=gradient_boosting_regressor.predict(x_test2)/100))
<train_model><EOS> 
predictions2 = gradient_boosting_regressor.predict(test)
 predictions = np.exp(( predictions1 + predictions2)/200)
 my_sub = pd.DataFrame({'Id':test2.Id, 'SalePrice':predictions})
 my_sub.to_csv('submission.csv', index = False)