Kaggle competition: House Prices - Advanced Regression Techniques
X_stack_val = val_ens_predictions
val_meta_predictions = metalearner.predict(X_stack_val)
 X_stack_val.shape, val_meta_predictions.shape<compute_test_metric>
import matplotlib
import matplotlib.pyplot as plt
from scipy.stats import skew
from sklearn.linear_model import ElasticNet
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from itertools import product
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.metrics import mean_squared_error
m.accuracy_score(val_meta_predictions, val.target.values )<predict_on_test>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
def get_test_predictions_using_metalearner(model, x, ids):
prediction = model.predict(x)
 result = np.round(prediction ).astype(int)
 output = pd.DataFrame({'id':ids,'target': result})
 return output<feature_engineering>
pred_df = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 pred_df['text'] = pred_df['text'].apply(lambda x: x.lower())
 pred_df['text'] = pred_df['text'].apply(lambda x: remove_url(x))
pred_df['text'] = pred_df['text'].apply(lambda x: remove_user(x))
pred_df['text'] = pred_df['text'].apply(lambda x: translate_with_mapping(x, synonyms_mapping))
pred_df['text'] = pred_df['text'].apply(lambda x: translate_with_mapping(x, abbreviations_mapping))
pred_df['text'] = pred_df['text'].apply(lambda x: translate_with_mapping(x, contraction_mapping))
pred_df['text'] = pred_df['text'].apply(lambda x: remove_punct_dup(x))<predict_on_test>
features = ['GrLivArea','OverallQual','2ndFlrSF','TotalBsmtSF','1stFlrSF','GarageCars','YearBuilt', 'GarageArea','ExterQual',
'YearRemodAdd','FireplaceQu','GarageYrBlt','FullBath','MSSubClass','LotArea','Fireplaces','KitchenQual','MSZoning',
'GarageType','BsmtFinSF1','Neighborhood','BsmtQual','TotRmsAbvGrd','HalfBath','BldgType','GarageFinish',
'Foundation','BedroomAbvGr','HouseStyle','CentralAir','OpenPorchSF','HeatingQC','BsmtFinType1','BsmtUnfSF',
'GarageCond','GarageQual','KitchenAbvGr','OverallCond','BsmtCond','MasVnrArea','BsmtFullBath','Exterior1st',
'Exterior2nd','PavedDrive','WoodDeckSF','LandContour','MasVnrType','BsmtFinType2','Functional','Fence']
pred_ens_predictions = get_predicitons_with_ensemble(pred_df.text.values, ensemble_of_classifiers )<save_to_csv>
train = train[features+['SalePrice']]
test = test[['Id']+features]
all_data = pd.concat(( train.loc[:,'GrLivArea':'Fence'],
test.loc[:,'GrLivArea':'Fence']))
get_test_predictions_using_metalearner(metalearner, pred_ens_predictions, pred_df.id ).to_csv('submission.csv', index=False )<set_options>
train['SalePrice'] = np.log1p(train['SalePrice'])
 numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = train[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
all_data[skewed_feats] = np.log1p(all_data[skewed_feats] )
warnings.filterwarnings("ignore")
 pd.options.display.max_colwidth = 170 
<load_from_csv>
for feature in all_data.select_dtypes(exclude=['object'] ).columns:
all_data[feature].fillna(all_data[feature].median() , inplace = True)
 for feature in all_data.select_dtypes(include=['object'] ).columns:
all_data[feature].fillna(all_data[feature].value_counts().idxmax() , inplace = True )
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
<count_values>
all_data = pd.get_dummies(all_data)
 print(all_data.shape )
train.target.value_counts()<randomize_order>
X_train = all_data[:train.shape[0]]
X_test = all_data[train.shape[0]:]
y = train.SalePrice
train = train.reindex(np.random.permutation(train.index))<define_variables>
def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, X_train, y, scoring="neg_mean_squared_error", cv = 5))
return(rmse )
train_len = train.shape[0]
test_len = test.shape[0]
train_keyword_null_count = train[train.keyword.isnull() == True].shape[0]
test_keyword_null_count = test[test.keyword.isnull() == True].shape[0]
train_location_null_count = train[train.location.isnull() == True].shape[0]
test_location_null_count = test[test.location.isnull() == True].shape[0]
print("Training data having {}% Keyword as Null".format(( train_keyword_null_count*100)/train_len))
print("Test data having {}% Keyword as Null
".format(( test_keyword_null_count*100)/test_len))
print("Training data having {}% Loaction as Null".format(train_location_null_count*100/train_len))
print("Test data having {}% Location as Null
".format(test_location_null_count*100/test_len))<count_missing_values>
alphas = [0.0005, 0.001, 0.01, 0.03, 0.05, 0.1]
l1_ratios = [1.5, 1.1, 1, 0.9, 0.8, 0.7, 0.5]
train.location.isnull().value_counts()<drop_column>
cv_elastic = [rmse_cv(ElasticNet(alpha = alpha, l1_ratio=l1_ratio)).mean()
for(alpha, l1_ratio)in product(alphas, l1_ratios)]
train.drop("location", axis = 1, inplace = True)
 test.drop("location", axis = 1, inplace = True )<string_transform>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
train.keyword.fillna("", inplace = True)
 test.keyword.fillna("", inplace = True)
 train.text = train.text + " " + train.keyword
test.text = test.text + " " + test.keyword<drop_column>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, X_train.values, y, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
train.drop("keyword", axis = 1, inplace = True)
 test.drop("keyword", axis = 1, inplace = True )<define_variables>
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
train_filter0 = train.target == 0
train_filter1 = train.target == 1<categorify>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
train["length"] = train.text.map(len)
 test["length"] = test.text.map(len )<feature_engineering>
ENet.fit(X_train.values, y)
 ENet_train_pred = ENet.predict(X_train.values)
 print('RMSLE score on train data:')
 print(rmsle(y, ENet_train_pred))
train["word_cnt"] = train.text.apply(lambda x : len(x.split(" ")))
 test["word_cnt"] = test.text.apply(lambda x : len(x.split(" ")))
 train["a_count"] = train.text.apply(lambda x : len([char for char in str(x)if char == "@"]))
test["a_count"] = test.text.apply(lambda x : len([char for char in str(x)if char == "@"]))
train["hash_count"] = train.text.apply(lambda x : len([char for char in str(x)if char == "
test["hash_count"] = test.text.apply(lambda x : len([char for char in str(x)if char == "<categorify>
ENet_pred = np.expm1(ENet.predict(X_test.values))
def dict_formation(data):

 dict_word = {}
for sent in data.text.tolist() :
words = sent.split(" ")
 for word in words:
word = word.lower()
try:
dict_word[word] = dict_word[word]+1
except:
dict_word[word] = 1
return dict_word<categorify>
sub = pd.DataFrame()
sub['Id'] = test['Id']
sub['SalePrice'] = ENet_pred
sub.to_csv('submission.csv',index=False )