Kaggle competition: House Prices - Advanced Regression Techniques
model = Sequential()
model.add(Conv2D(16, 3, activation='relu', padding='same', input_shape=(im_size, im_size, 1)))
 model.add(Dropout(0.5))
model.add(Conv2D(16, 3, activation='relu', padding='same'))
model.add(Dropout(0.5))
model.add(Conv2D(16, 5, activation='relu', padding='same'))
model.add(MaxPooling2D())
 model.add(Conv2D(32, 3, activation='relu', padding='same'))
model.add(Dropout(0.5))
model.add(Conv2D(32, 3, activation='relu', padding='same'))
model.add(Dropout(0.5))
model.add(Conv2D(32, 5, activation='relu', padding='same'))
model.add(MaxPooling2D())
 model.add(Conv2D(64, 3, activation='relu', padding='same'))
model.add(Dropout(0.5))
model.add(Conv2D(64, 3, activation='relu', padding='same'))
model.add(Dropout(0.5))
model.add(Conv2D(64, 5, activation='relu', padding='same'))
model.add(MaxPooling2D())
 model.add(Flatten())
 model.add(Dense(64, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
model.summary()
model.compile(
 loss='categorical_crossentropy',
optimizer='adam',
metrics=['acc']
)<set_options>
warnings.filterwarnings('ignore')
 % matplotlib inline
imagegen = ImageDataGenerator(
 rotation_range=15,
width_shift_range=0.2,
height_shift_range=0.2,
zoom_range=0.2
)<train_model>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
se_callback = SnapshotEnsemble(n_models=7, n_epochs_per_model=50, lr_max=.001)
 history = model.fit_generator(
 imagegen.flow(x_train, y_train, batch_size=32),
steps_per_epoch=len(x_train)/ 32,
epochs=se_callback.n_epochs_total,
verbose=0,
callbacks=[se_callback],
validation_data=(x_test, y_test)
 )<predict_on_test>
target = np.log(train.SalePrice)
 print("Skewness : %f" % target.skew())
 print("Kurtosis : %f" % target.kurt() )
def predict(models, data, weights=None):
if weights is None:
weights = [1 /(len(models)) ] * len(models)
 pred = np.zeros(( data.shape[0], 10))
for i, model in enumerate(models):
pred += model.predict(data)* weights[i]
return pred
def evaluate(preds, weights=None):
if weights is None:
weights = [1 / len(preds)] * len(preds)
 y_pred = np.zeros(( y_test.shape[0], 10))
for i, pred in enumerate(preds):
y_pred += pred * weights[i]
y_pred = np.argmax(y_pred, axis=1)
 y_true = np.argmax(y_test, axis=1)
 return accuracy_score(y_true, y_pred)
 models = se_callback.load_ensemble()
preds = []
for i, model in enumerate(models):
pred = predict([model], x_test)
 preds.append(pred)
 score = evaluate([pred])
 print(f'model {i + 1}: accuracy = {score:.4f}')
 ensemble_score = evaluate(preds)
 print(f'ensemble: accuracy = {ensemble_score:.4f}' )<find_best_params>
train = train[train['GarageArea'] < 1200]
best_score = ensemble_score
best_weights = None
no_improvements = 0
while no_improvements < 5000:
new_weights = np.random.uniform(size=(len(models),))
new_weights /= new_weights.sum()
new_score = evaluate(preds, new_weights)
 if new_score > best_score:
no_improvements = 0
best_score = new_score
best_weights = new_weights
print(f'improvement: {best_score:.4f}')
 else:
no_improvements += 1
print(f'best weights are {best_weights}' )<save_to_csv>
total = train.isnull().sum().sort_values(ascending = False)
 percent =(train.isnull().sum() /train.isnull().count() ).sort_values(ascending = False)
 missing_data = pd.concat([total,percent], axis = 1, keys = ['Total', 'Percent'])
 missing_data.head(20 )
pred = predict(models, test, best_weights)
 res = pd.DataFrame()
res['ImageId'] = np.arange(test.shape[0])+ 1
res['Label'] = np.argmax(pred, axis=1)
 res.to_csv('submission.csv', index=False)
 res.head(15 )<load_from_csv>
print("Original:
")
 print(train.Street.value_counts() , "
" )
tweet = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv' )<string_transform>
train['enc_street'] = pd.get_dummies(train.Street, drop_first=True)
 test['enc_street'] = pd.get_dummies(train.Street, drop_first=True )
def create_corpus(target):
corpus=[]
for x in tweet[tweet['target']==target]['text'].str.split() :
for i in x:
corpus.append(i)
 return corpus<string_transform>
def encode(x): return 1 if x == 'Partial' else 0
train['enc_condition'] = train.SaleCondition.apply(encode)
 test['enc_condition'] = test.SaleCondition.apply(encode )
def create_corpus_df(tweet, target):
corpus=[]
for x in tweet[tweet['target']==target]['text'].str.split() :
for i in x:
corpus.append(i)
 return corpus<concatenate>
def encode(x): return 1 if x == 'Ex' else 0
train['enc_kitchen'] = train.SaleCondition.apply(encode)
 test['enc_kitchen'] = test.SaleCondition.apply(encode )
df = pd.concat([tweet,test])
 df.shape<categorify>
train['enc_utilities'] = pd.get_dummies(train.Utilities, drop_first=True)
 test['enc_utilities'] = pd.get_dummies(train.Utilities, drop_first=True )
df['location'].replace({'United States':'USA',
'New York':'USA',
"London":'UK',
"Los Angeles, CA":'USA',
"Washington, D.C.":'USA',
"California":'USA',
"Chicago, IL":'USA',
"Chicago":'USA',
"New York, NY":'USA',
"California, USA":'USA',
"FLorida":'USA',
"Nigeria":'Africa',
"Kenya":'Africa',
"Everywhere":'Worldwide',
"San Francisco":'USA',
"Florida":'USA',
"United Kingdom":'UK',
"Los Angeles":'USA',
"Toronto":'Canada',
"San Francisco, CA":'USA',
"NYC":'USA',
"Seattle":'USA',
"Earth":'Worldwide',
"Ireland":'UK',
"London, England":'UK',
"New York City":'USA',
"Texas":'USA',
"London, UK":'UK',
"Atlanta, GA":'USA',
"Mumbai":"India"},inplace=True)
 sns.barplot(y = df['location'].value_counts() [:5].index, x = df['location'].value_counts() [:5],
palette='autumn', orient='h' )<feature_engineering>
def encode(x): return 1 if x == 'Con' else 0
train['enc_type'] = train.SaleType.apply(encode)
 test['enc_type'] = test.SaleType.apply(encode )
def clean_text(text):

 text = text.lower()
text = re.sub('\[.*?\]', '', text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 text = re.sub('\w*\d\w*', '', text)
 return text
df['text'] = df['text'].apply(lambda x: clean_text(x))
df['text'].head()<drop_column>
y = np.log(train.SalePrice)
 X = data.drop(['SalePrice', 'Id'], axis=1 )
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 df['text']=df['text'].apply(lambda x: remove_emoji(x))<prepare_x_and_y>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
tweet_1 = tweet.text.values
test_1 = test.text.values
sentiments = tweet.target.values<train_model>
regr = linear_model.LinearRegression()
regr.fit(X_train, y_train)
 y_pred = regr.predict(X_test)
 print('The accuracy of the Linear Regression is',r2_score(y_test,y_pred))
print('RMSE is:
', mean_squared_error(y_test, y_pred))
word_tokenizer = Tokenizer()
word_tokenizer.fit_on_texts(tweet_1)
 vocab_length = len(word_tokenizer.word_index)+ 1<compute_test_metric>
xg_reg = xgb.XGBRegressor(
 colsample_bytree=0.2,
gamma=0.0,
learning_rate=0.05,
max_depth=6,
min_child_weight=1.5,
n_estimators=7200,
reg_alpha=0.9,
reg_lambda=0.6,
subsample=0.2,
seed=42,
silent=1)
 xg_reg.fit(X_train,y_train)
 preds = xg_reg.predict(X_test)
 print('The accuracy of the xgboost is',r2_score(y_test,preds))
print('RMSE is:
', mean_squared_error(y_test,preds))
def metrics(pred_tag, y_test):
print("F1-score: ", f1_score(pred_tag, y_test))
print("Precision: ", precision_score(pred_tag, y_test))
print("Recall: ", recall_score(pred_tag, y_test))
print("Acuracy: ", accuracy_score(pred_tag, y_test))
print("-"*50)
 print(classification_report(pred_tag, y_test))
def embed(corpus):
return word_tokenizer.texts_to_sequences(corpus )<string_transform>
gbr_model = GradientBoostingRegressor().fit(X_train,y_train)
 gbr_preds = gbr_model.predict(X_test)
 print('The accuracy of the Gradient boost is',r2_score(y_test,gbr_preds))
print('RMSE is:
', mean_squared_error(y_test,gbr_preds))
longest_train = max(tweet_1, key=lambda sentence: len(word_tokenize(sentence)))
 length_long_sentence = len(word_tokenize(longest_train))
padded_sentences = pad_sequences(embed(tweet_1), length_long_sentence, padding='post')
 test_sentences = pad_sequences(
 embed(test_1),
length_long_sentence,
padding='post'
)<feature_engineering>
X_train=pd.get_dummies(X_train)
 X_test=pd.get_dummies(X_test)
 model = BaggingRegressor(tree.DecisionTreeRegressor(random_state=1))
model.fit(X_train, y_train)
 bag_preds = model.predict(X_test)
 print('The accuracy of the Bagging Regressor is',r2_score(y_test,bag_preds))
print('RMSE is:
', mean_squared_error(y_test,bag_preds))
embeddings_dictionary = dict()
embedding_dim = 100
glove_file = open('.. /input/glove-file/glove.6B.100d.txt')
 for line in glove_file:
records = line.split()
word = records[0]
vector_dimensions = np.asarray(records[1:], dtype='float32')
 embeddings_dictionary [word] = vector_dimensions
glove_file.close()<feature_engineering>
lgbm_model = LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6,
min_sum_hessian_in_leaf = 11 ).fit(X_train, y_train)
 lgbm_preds = lgbm_model.predict(X_test)
 print('The accuracy of the Bagging Regressor is',r2_score(y_test,lgbm_preds))
print('RMSE is:
', mean_squared_error(y_test,lgbm_preds))
embedding_matrix = np.zeros(( vocab_length, embedding_dim))
for word, index in word_tokenizer.word_index.items() :
embedding_vector = embeddings_dictionary.get(word)
 if embedding_vector is not None:
embedding_matrix[index] = embedding_vector<split>
r_alphas = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
ridge_model = make_pipeline(RobustScaler() , RidgeCV(alphas = r_alphas)).fit(X_train, y_train)
 ridge_preds = ridge_model.predict(X_test)
 print('The accuracy of the ridge Regressor is',r2_score(y_test,ridge_preds))
print('RMSE is:
', mean_squared_error(y_test,ridge_preds))
X_train, X_test, y_train, y_test = train_test_split(
 padded_sentences,
sentiments,
test_size=0.25
)<choose_model_class>
alphas= [0.00005, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005,
0.0006, 0.0007, 0.0008]
lasso_model = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e7,
alphas = alphas,
random_state = 42)).fit(X_train, y_train)
 lasso_preds = lasso_model.predict(X_test)
 print('The accuracy of the lasso Regressor is',r2_score(y_test,lasso_preds))
print('RMSE is:
', mean_squared_error(y_test,lasso_preds))
def BLSTM() :
model = Sequential()
model.add(Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
weights = [embedding_matrix],
input_length=length_long_sentence))
model.add(Bidirectional(LSTM(length_long_sentence, return_sequences = True, recurrent_dropout=0.2)))
 model.add(GlobalMaxPool1D())
 model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(length_long_sentence, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(length_long_sentence, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(1, activation = 'sigmoid'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
 return model<train_model>
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
elastic_cv = make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas,
l1_ratio=e_l1ratio))
elastic_model = elastic_cv.fit(X_train, y_train)
 elastic_preds = elastic_model.predict(X_test)
 print('The accuracy of the lasso Regressor is',r2_score(y_test,elastic_preds))
print('RMSE is:
', mean_squared_error(y_test,elastic_preds))
model = BLSTM()
checkpoint = ModelCheckpoint(
 'model.h5',
monitor = 'val_loss',
verbose = 1,
save_best_only = True 
)
 reduce_lr = ReduceLROnPlateau(
 monitor = 'val_loss',
factor = 0.2,
verbose = 1,
patience = 5,
min_lr = 0.001 
)
 history = model.fit(
 X_train,
y_train,
epochs = 7,
batch_size = 32,
validation_data = [X_test, y_test],
verbose = 1,
callbacks = [reduce_lr, checkpoint]
)<compute_test_metric>
ridge = make_pipeline(RobustScaler() ,
RidgeCV(alphas = r_alphas))
lasso = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e7, alphas = alphas,
random_state = 42))
elasticnet = make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas,
l1_ratio=e_l1ratio))
lgbm_model = LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6,
min_sum_hessian_in_leaf = 11 ).fit(X_train,y_train)
 gbr_model = GradientBoostingRegressor().fit(X_train,y_train)
 stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet, gbr_model,
lgbm_model),
meta_regressor=gbr_model,
use_features_in_secondary=True)
 stackX = np.array(X_train)
 stacky = np.array(y_train)
 stack_gen_model = stack_gen.fit(stackX, stacky )
loss, accuracy = model.evaluate(X_test, y_test)
 print('Loss:', loss)
 print('Accuracy:', accuracy )<predict_on_test>
em_preds = elastic_model.predict(X_test)
 lasso_preds = lasso_model.predict(X_test)
 ridge_preds = ridge_model.predict(X_test)
 stack_gen_preds = stack_gen_model.predict(X_test)
 lgbm_preds = lgbm_model.predict(X_test)
 gbr_preds = gbr_model.predict(X_test )
preds = model.predict_classes(X_test)
 metrics(preds, y_test )<predict_on_test>
stack_preds =(( 0.1*em_preds)+(0.2*lasso_preds)+(0.1*ridge_preds)+(0.2 * gbr_preds)
 +(0.1*lgbm_preds)+(0.3*stack_gen_preds))
print('The accuracy of the stack Regressor is',r2_score(y_test,stack_preds))
print('RMSE is:
', mean_squared_error(y_test,stack_preds))
model.load_weights('model.h5')
 preds = model.predict_classes(X_test)
 metrics(preds, y_test )<save_to_csv>
feats = test.select_dtypes(include=[np.number] ).interpolate().dropna()
feats = feats.drop([ 'Id'], axis=1 )
submission.target = model.predict_classes(test_sentences)
 submission.to_csv("submission.csv", index=False)
 submission.target.value_counts().plot.bar() ;<import_modules>
em_preds = elastic_model.predict(feats)
 lasso_preds = lasso_model.predict(feats)
 ridge_preds = ridge_model.predict(feats)
 stack_gen_preds = stack_gen_model.predict(feats)
 lgbm_preds = lgbm_model.predict(feats)
 gbr_preds = gbr_model.predict(feats )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from keras.models import Sequential
from keras.layers import Conv2D, Lambda, MaxPooling2D
from keras.layers import Dense, Dropout, Flatten
from keras.layers.normalization import BatchNormalization
from keras.preprocessing.image import ImageDataGenerator
from keras.utils.np_utils import to_categorical 
<load_from_csv>
stack_preds =(( 0.1*em_preds)+(0.2*lasso_preds)+(0.1*ridge_preds)+(0.2 * gbr_preds)
 +(0.1*lgbm_preds)+(0.3*stack_gen_preds))
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv')
 print("Data are Ready!!" )<train_model>
final_predictions = np.exp(stack_preds)
 print("Original predictions are:
", stack_preds[:5], "
")
 print("Final predictions are:
", final_predictions[:5] )
<prepare_x_and_y><EOS> 
submission = pd.DataFrame()
submission['Id'] = test.Id
submission['SalePrice'] = final_predictions
submission.to_csv('submission1.csv', index=False )