Kaggle competition: House Prices - Advanced Regression Techniques
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()<sort_values>
%matplotlib inline
sns.set_style('whitegrid' )
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )<feature_engineering>
import scipy.stats as stats
from scipy import stats
from scipy.stats import pointbiserialr, spearmanr, skew, pearsonr
df = df1[['Patient_Week']].copy()
df['FVC'] = 0.25*df1['FVC'] + 0.75*df2['FVC']
df['Confidence'] = 0.26*df1['Confidence'] + 0.74*df2['Confidence']
df.head()<save_to_csv>
from sklearn.ensemble import RandomForestRegressor
from sklearn.feature_selection import SelectFromModel
from sklearn.model_selection import cross_val_score, train_test_split, GridSearchCV
from sklearn.linear_model import Ridge, RidgeCV, LassoCV
from sklearn import linear_model
df.to_csv('submission.csv', index=False )<set_options>
house_train = pd.read_csv(".. /input/train.csv")
 house_test = pd.read_csv(".. /input/test.csv" )
%matplotlib inline
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 200
MAX_FEATURES = 120000
IDENTITY_COLUMNS = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness'
]
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'<feature_engineering>
house_train[['OverallQual','SalePrice']].groupby(['OverallQual'],
as_index=False ).mean().sort_values(by='OverallQual', ascending=False )
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix<load_from_csv>
house_train[['GarageCars','SalePrice']].groupby(['GarageCars'],
as_index=False ).mean().sort_values(by='GarageCars', ascending=False )
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_train = train_df[TEXT_COLUMN].astype(str)
 y_aux_train = train_df[AUX_COLUMNS].values
y_train = train_df[TARGET_COLUMN].values
x_test = test_df[TEXT_COLUMN].astype(str )<feature_engineering>
house_train[['Fireplaces','SalePrice']].groupby(['Fireplaces'],
as_index=False ).mean().sort_values(by='Fireplaces', ascending=False )
for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >= 0.5, True, False)
 tokenizer = text.Tokenizer(num_words=MAX_FEATURES, filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test )<string_transform>
house_train.isnull().sum().sort_values(ascending=False ).head(20 )
x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<define_variables>
house_test.isnull().sum().sort_values(ascending=False ).head(20 )
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1 )<choose_model_class>
total = house_train.isnull().sum().sort_values(ascending=False)
 percent =(house_train.isnull().sum() /house_train.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(20 )
def build_model(embedding_matrix, num_aux_targets):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<define_variables>
total = house_test.isnull().sum().sort_values(ascending=False)
 percent =(house_test.isnull().sum() /house_test.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(20 )
checkpoint_predictions = []<train_model>
to_null = ['Alley', 'MasVnrType', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'FireplaceQu',
'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'GarageYrBlt', 'BsmtFullBath', 'BsmtHalfBath',
'PoolQC', 'Fence', 'MiscFeature']
for col in to_null:
if house_train[col].dtype == 'object':
house_train[col].fillna('None',inplace=True)
 house_test[col].fillna('None',inplace=True)
 else:
house_train[col].fillna(0,inplace=True)
 house_test[col].fillna(0,inplace=True )
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=1,
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten() )<prepare_output>
house_test.loc[house_test.KitchenQual.isnull() , 'KitchenQual'] = 'TA'
house_test.loc[house_test.MSZoning.isnull() , 'MSZoning'] = 'RL'
house_test.loc[house_test.Utilities.isnull() , 'Utilities'] = 'AllPub'
house_test.loc[house_test.Exterior1st.isnull() , 'Exterior1st'] = 'VinylSd'
house_test.loc[house_test.Exterior2nd.isnull() , 'Exterior2nd'] = 'VinylSd'
house_test.loc[house_test.Functional.isnull() , 'Functional'] = 'Typ'
house_test.loc[house_test.SaleType.isnull() , 'SaleType'] = 'WD'
house_train.loc[house_train['Electrical'].isnull() , 'Electrical'] = 'SBrkr'
house_train.loc[house_train['LotFrontage'].isnull() , 'LotFrontage'] = house_train['LotFrontage'].mean()
house_test.loc[house_test['LotFrontage'].isnull() , 'LotFrontage'] = house_test['LotFrontage'].mean()
predictions = np.average(checkpoint_predictions, axis=0 )<save_to_csv>
house_train.loc[house_train.MasVnrType == 'None', 'MasVnrArea'] = 0
house_test.loc[house_test.MasVnrType == 'None', 'MasVnrArea'] = 0
house_test.loc[house_test.BsmtFinType1=='None', 'BsmtFinSF1'] = 0
house_test.loc[house_test.BsmtFinType2=='None', 'BsmtFinSF2'] = 0
house_test.loc[house_test.BsmtQual=='None', 'BsmtUnfSF'] = 0
house_test.loc[house_test.BsmtQual=='None', 'TotalBsmtSF'] = 0
submission = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 submission['prediction'] = predictions
submission.to_csv('submission.csv', index=False )<import_modules>
house_test.loc[house_test.GarageCars.isnull() , 'GarageCars'] = 0
house_test.loc[house_test.GarageArea.isnull() , 'GarageArea'] = 0
import numpy as np
import pandas as pd
from keras.models import Model
from keras.layers import Input, Dense, Embedding, SpatialDropout1D, add, concatenate
from keras.layers import CuDNNLSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.preprocessing import text, sequence
from keras.callbacks import LearningRateScheduler<set_options>
total = house_test.isnull().sum().sort_values(ascending=False)
 percent =(house_test.isnull().sum() /house_test.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(30 )
from sklearn.model_selection import train_test_split 
<define_variables>
threshold = 0.8
def correlation() :
for i in house_train.columns:
for j in house_train.columns[list(house_train.columns ).index(i)+ 1:]:
if house_train[i].dtype != 'object' and house_train[j].dtype != 'object':
if abs(pearsonr(house_train[i], house_train[j])[0])>= threshold:
yield(pearsonr(house_train[i], house_train[j])[0], i, j)
 else:
if abs(spearmanr(house_train[i], house_train[j])[0])>= threshold:
yield(spearmanr(house_train[i], house_train[j])[0], i, j )
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
NUM_MODELS = 4
BATCH_SIZE = 330
LSTM_UNITS = 120
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 3
MAX_LEN = 210
IDENTITY_COLUMNS = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness'
]
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'<compute_test_metric>
house_train.loc[house_train.PoolArea == 555]
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32' )<load_pretrained>
house_train['MSSubClass'] = house_train['MSSubClass'].astype(str)
 house_test['MSSubClass'] = house_test['MSSubClass'].astype(str )
def load_embeddings(path):
with open(path,'r', encoding='UTF-8')as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f )<categorify>
for col in house_train.columns:
if house_train[col].dtype != 'object':
if skew(house_train[col])> 0.75:
house_train[col] = np.log1p(house_train[col])
 pass
else:
dummies = pd.get_dummies(house_train[col], drop_first=False)
 dummies = dummies.add_prefix("{}_".format(col))
house_train.drop(col, axis=1, inplace=True)
 house_train = house_train.join(dummies)
 for col in house_test.columns:
if house_test[col].dtype != 'object':
if skew(house_test[col])> 0.75:
house_test[col] = np.log1p(house_test[col])
 pass
else:
dummies = pd.get_dummies(house_test[col], drop_first=False)
 dummies = dummies.add_prefix("{}_".format(col))
house_test.drop(col, axis=1, inplace=True)
 house_test = house_test.join(dummies )
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix<load_from_csv>
X_train = house_train.drop('SalePrice',axis=1)
 Y_train = house_train['SalePrice']
X_test = house_test
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )<prepare_x_and_y>
def rmlse(val, target):
return np.sqrt(np.sum(((np.log1p(val)- np.log1p(np.expm1(target)))**2)/ len(target)) )
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str )<feature_engineering>
Xtrain, Xtest, ytrain, ytest = train_test_split(X_train, Y_train, test_size=0.30 )
for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >= 0.55, True, False )<string_transform>
ridge = Ridge(alpha=10, solver='auto' ).fit(Xtrain, ytrain)
 val_ridge = np.expm1(ridge.predict(Xtest))
rmlse(val_ridge, ytest )
tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))<string_transform>
ridge_cv = RidgeCV(alphas=(0.01, 0.05, 0.1, 0.3, 1, 3, 5, 10))
ridge_cv.fit(Xtrain, ytrain)
 val_ridge_cv = np.expm1(ridge_cv.predict(Xtest))
rmlse(val_ridge_cv, ytest )
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<define_variables>
las = linear_model.Lasso(alpha=0.0005 ).fit(Xtrain, ytrain)
 las_ridge = np.expm1(las.predict(Xtest))
rmlse(las_ridge, ytest )
sample_weights = np.ones(len(x_train), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights += train_df[TARGET_COLUMN] *(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])* train_df[IDENTITY_COLUMNS].sum(axis=1)* 5
sample_weights /= sample_weights.mean()<concatenate>
las_cv = LassoCV(alphas=(0.0001, 0.0005, 0.001, 0.01, 0.05, 0.1, 0.3, 1, 3, 5, 10))
las_cv.fit(Xtrain, ytrain)
 val_las_cv = np.expm1(las_cv.predict(Xtest))
rmlse(val_las_cv, ytest )
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1)
<split>
model_xgb = xgb.XGBRegressor(n_estimators=340, max_depth=2, learning_rate=0.2)
 model_xgb.fit(Xtrain, ytrain)
 xgb_preds = np.expm1(model_xgb.predict(Xtest))
rmlse(xgb_preds, ytest )
X, X_t, y, y_t , ya, ya_t, sw, sw_t = train_test_split(x_train, y_train , y_aux_train ,sample_weights, test_size=0.18, random_state=42 )<choose_model_class>
forest = RandomForestRegressor(min_samples_split =5,
min_weight_fraction_leaf = 0.0,
max_leaf_nodes = None,
max_depth = None,
n_estimators = 300,
max_features = 'auto')
 forest.fit(Xtrain, ytrain)
 Y_pred_RF = np.expm1(forest.predict(Xtest))
rmlse(Y_pred_RF, ytest )
def build_model(embedding_matrix, num_aux_targets):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])
 return model<choose_model_class>
X_test['RoofMatl_ClyTile'] = 0
model = build_model(embedding_matrix, ya.shape[-1] )<define_variables>
X = X_train[features]
Xt = X_test[features]
checkpoint_predictions = []<train_model>
Xtrain1, Xtest1, ytrain1, ytest1 = train_test_split(X, Y_train, test_size=0.33 )
for global_model in range(NUM_MODELS):
model.fit(X, [y, ya], validation_data=[X_t, [y_t,ya_t]],
batch_size=BATCH_SIZE, epochs= 2, verbose=2,sample_weight=[sw.values, np.ones_like(sw)],
callbacks=[LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_model)) ])
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten() )<prepare_output>
ridge = Ridge(alpha=5, solver='svd' ).fit(Xtrain1, ytrain1)
 val_ridge = np.expm1(ridge.predict(Xtest1))
rmlse(val_ridge, ytest1 )
predictions = np.average(checkpoint_predictions, axis=0 )<save_to_csv>
las_cv = LassoCV(alphas=(0.0001, 0.0005, 0.001, 0.01, 0.05, 0.1, 0.3, 1, 3, 5, 10)).fit(Xtrain1, ytrain1)
 val_las = np.expm1(las_cv.predict(Xtest1))
rmlse(val_las, ytest1 )
submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions
})
 submission.to_csv('submission.csv', index=False )<load_from_csv>
model_xgb = xgb.XGBRegressor(n_estimators=340, max_depth=2, learning_rate=0.2)
 model_xgb.fit(Xtrain1, ytrain1)
 xgb_preds = np.expm1(model_xgb.predict(Xtest1))
rmlse(xgb_preds, ytest1 )
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )<drop_column>
forest = RandomForestRegressor(min_samples_split =5,
min_weight_fraction_leaf = 0.0,
max_leaf_nodes = None,
max_depth = 100,
n_estimators = 300,
max_features = None)
 forest.fit(Xtrain1, ytrain1)
 Y_pred_RF = np.expm1(forest.predict(Xtest1))
rmlse(Y_pred_RF, ytest1 )
identities = ['male','female','transgender','other_gender','heterosexual','homosexual_gay_or_lesbian',
'bisexual','other_sexual_orientation','christian','jewish','muslim','hindu','buddhist',
'atheist','other_religion','black','white','asian','latino','other_race_or_ethnicity',
'physical_disability','intellectual_or_learning_disability','psychiatric_or_mental_illness',
'other_disability']
train_labeled_df = train.loc[:, ['target'] + identities ].dropna()
toxic_df = train_labeled_df[train_labeled_df['target'] >=.5][identities]
non_toxic_df = train_labeled_df[train_labeled_df['target'] <.5][identities]
toxic_count = toxic_df.where(train_labeled_df == 0, other = 1 ).sum()
non_toxic_count = non_toxic_df.where(train_labeled_df == 0, other = 1 ).sum()
toxic_vs_non_toxic = pd.concat([toxic_count, non_toxic_count], axis=1)
 toxic_vs_non_toxic = toxic_vs_non_toxic.rename(index=str, columns={1: "non-toxic", 0: "toxic"})
 toxic_vs_non_toxic.sort_values(by='toxic' ).plot(kind='bar', stacked=True, figsize=(30,10), fontsize=20 ).legend(prop={'size': 20} )<train_model>
las_cv1 = LassoCV(alphas=(0.0001, 0.0005, 0.001, 0.01, 0.05, 0.1, 0.3, 1, 3, 5, 10))
las_cv1.fit(X, Y_train)
 lasso_preds = np.expm1(las_cv1.predict(Xt))
from fastai.train import Learner
from fastai.train import DataBunch
from fastai.callbacks import *
from fastai.basic_data import DatasetType<import_modules>
model_xgb = xgb.XGBRegressor(n_estimators=340, max_depth=2, learning_rate=0.1)
 model_xgb.fit(X, Y_train)
 xgb_preds = np.expm1(model_xgb.predict(Xt))
import numpy as np
import pandas as pd
import os
import time
import gc
import random
from tqdm._tqdm_notebook import tqdm_notebook as tqdm
from keras.preprocessing import text, sequence
import torch
from torch import nn
from torch.utils import data
from torch.nn import functional as F<set_options>
preds = 0.7 * lasso_preds + 0.3 * xgb_preds
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<define_variables>
submission = pd.DataFrame({
'Id': house_test['Id'].astype(int),
'SalePrice': preds
})
 submission.to_csv('home.csv', index=False )
CRAWL_EMBEDDING_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
GLOVE_EMBEDDING_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220 
<compute_test_metric>
model_lasso = LassoCV(alphas=(0.0001, 0.0005, 0.001, 0.01, 0.05, 0.1, 0.3, 1, 3, 5, 10, 100))
model_lasso.fit(X_train, Y_train)
 coef = pd.Series(model_lasso.coef_, index = X_train.columns)
 v1 = coef.loc[model_lasso.coef_ != 0].count()
print('So we have ' + str(v1)+ ' variables' )
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f)) 
<categorify>
for i in features_f:
if i not in X_test.columns:
X_test[i] = 0
print(i)
 X = X_train[features_f]
Xt = X_test[features_f]
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words<train_model>
model_lasso = LassoCV(alphas=(0.0001, 0.0005, 0.001, 0.01, 0.05, 0.1, 0.3, 1, 3, 5, 10))
model_lasso.fit(X, Y_train)
 lasso_preds = np.expm1(model_lasso.predict(Xt))
def sigmoid(x):
return 1 /(1 + np.exp(-x))
def train_model(learn,test,output_dim,lr=0.001,
batch_size=512, n_epochs=4,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 n = len(learn.data.train_dl)
 phases = [(TrainingPhase(n ).schedule_hp('lr', lr *(0.6**(i)))) for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
 learn.callbacks.append(sched)
 for epoch in range(n_epochs):
learn.fit(1)
 test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(learn.model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds<normalization>
model_xgb = xgb.XGBRegressor(n_estimators=340, max_depth=2, learning_rate=0.1)
 model_xgb.fit(X, Y_train)
 xgb_preds = np.expm1(model_xgb.predict(Xt))
<categorify><EOS> 
solution = pd.DataFrame({"id":house_test.Id, "SalePrice":0.7*lasso_preds + 0.3*xgb_preds})
 solution.to_csv("House_price.csv", index = False )