Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<set_options>
warnings.filterwarnings('ignore')
 %matplotlib inline
rcParams['figure.figsize'] = 10,10<load_from_csv>
df_train = pd.read_csv('.. /input/train.csv' )
data = pd.read_csv('.. /input/train.csv')
 parent_data = data.copy()
ID = data.pop('id' )<categorify>
total = df_train.isnull().sum().sort_values(ascending=False)
 percent =(df_train.isnull().sum() /df_train.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(20 )
y = data.pop('species')
 y = LabelEncoder().fit(y ).transform(y)
 print(y.shape )<normalization>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print("The train data size before dropping Id feature is : {} ".format(train.shape))
print("The test data size before dropping Id feature is : {} ".format(test.shape))
train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True)
 print("
The train data size after dropping Id feature is : {} ".format(train.shape))
print("The test data size after dropping Id feature is : {} ".format(test.shape))
X = StandardScaler().fit(data ).transform(data)
 print(X.shape )<categorify>
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))
y_cat = to_categorical(y)
 print(y_cat.shape )<choose_model_class>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head(20 )
model = Sequential()
model.add(Dense(1024,input_dim=192, init='uniform', activation='relu'))
model.add(Dense(99, activation='softmax'))<choose_model_class>
all_data["PoolQC"] = all_data["PoolQC"].fillna("None" )
model.compile(loss='categorical_crossentropy',optimizer='adam', metrics = ["accuracy"] )<train_model>
all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None" )
history = model.fit(X,y_cat,batch_size=32,
nb_epoch=400,verbose=0 )<load_from_csv>
all_data["Alley"] = all_data["Alley"].fillna("None" )
test = pd.read_csv('.. /input/test.csv' )<drop_column>
all_data["Fence"] = all_data["Fence"].fillna("None" )
index = test.pop('id' )<normalization>
all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None" )
test = StandardScaler().fit(test ).transform(test )<predict_on_test>
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
yPred = model.predict_proba(test )<create_dataframe>
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None' )
yPred = pd.DataFrame(yPred,index=index,columns=sort(parent_data.species.unique()))<save_to_csv>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0 )
fp = open('submission_nn_kernel.csv','w')
 fp.write(yPred.to_csv() )<import_modules>
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0 )
!pip install -q tf-models-official==2.3.0 
<load_from_csv>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None' )
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test=pd.read_csv('.. /input/nlp-getting-started/test.csv')
 submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv" )<feature_engineering>
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )
train['words'] = train['text'].str.split()
train['word_len'] = train['words'].map(lambda x: len(x))
train['word_len'].max()<feature_engineering>
all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0] )
test['words'] = test['text'].str.split()
test['word_len'] = test['words'].map(lambda x: len(x))
test['word_len'].max()<feature_engineering>
all_data["Functional"] = all_data["Functional"].fillna("Typ" )
ids_with_target_error = [328,443,513,2619,3640,3900,4342,5781,6552,6554,6570,6701,6702,6729,6861,7226]
train.loc[train['id'].isin(ids_with_target_error),'target'] = 0
train[train['id'].isin(ids_with_target_error)]<concatenate>
all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0] )
df_all = pd.concat([train,test])
 df_all.shape<feature_engineering>
all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0] )
df_all['text'] = df_all['text'].str.lower()
df_all['text'].head(2 )<feature_engineering>
all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0] )
def remove_breaklines(text):
return re.sub('
','',text)
 df_all['text'] = df_all['text'].apply(lambda x: remove_breaklines(x))<feature_engineering>
all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )
def remove_numbers(text):
return re.sub('\w*\d\w*', '', text)
 df_all['text'] = df_all['text'].apply(lambda x: remove_numbers(x))<feature_engineering>
all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None" )
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 df_all['text'] = df_all['text'].apply(lambda x: remove_URL(x))<feature_engineering>
all_data = all_data.drop(['Utilities'], axis=1)
 all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head()
def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 df_all['text']=df_all['text'].apply(lambda x : remove_html(x))<drop_column>
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 df_all['text']=df_all['text'].apply(lambda x: remove_emoji(x))<feature_engineering>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))
all_data.head()
def remove_punct(text):
table=str.maketrans('','',string.punctuation)
 return text.translate(table)
 df_all['text']=df_all['text'].apply(lambda x : remove_punct(x))<define_variables>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
abbreviations = {
"$" : " dollar ",
"€" : " euro ",
"4ao" : "for adults only",
"a.m" : "before midday",
"a3" : "anytime anywhere anyplace",
"aamof" : "as a matter of fact",
"acct" : "account",
"adih" : "another day in hell",
"afaic" : "as far as i am concerned",
"afaict" : "as far as i can tell",
"afaik" : "as far as i know",
"afair" : "as far as i remember",
"afk" : "away from keyboard",
"app" : "application",
"approx" : "approximately",
"apps" : "applications",
"asap" : "as soon as possible",
"asl" : "age, sex, location",
"atk" : "at the keyboard",
"ave." : "avenue",
"aymm" : "are you my mother",
"ayor" : "at your own risk",
"b&b" : "bed and breakfast",
"b+b" : "bed and breakfast",
"b.c" : "before christ",
"b2b" : "business to business",
"b2c" : "business to customer",
"b4" : "before",
"b4n" : "bye for now",
"b@u" : "back at you",
"bae" : "before anyone else",
"bak" : "back at keyboard",
"bbbg" : "bye bye be good",
"bbc" : "british broadcasting corporation",
"bbias" : "be back in a second",
"bbl" : "be back later",
"bbs" : "be back soon",
"be4" : "before",
"bfn" : "bye for now",
"blvd" : "boulevard",
"bout" : "about",
"brb" : "be right back",
"bros" : "brothers",
"brt" : "be right there",
"bsaaw" : "big smile and a wink",
"btw" : "by the way",
"bwl" : "bursting with laughter",
"c/o" : "care of",
"cet" : "central european time",
"cf" : "compare",
"cia" : "central intelligence agency",
"csl" : "can not stop laughing",
"cu" : "see you",
"cul8r" : "see you later",
"cv" : "curriculum vitae",
"cwot" : "complete waste of time",
"cya" : "see you",
"cyt" : "see you tomorrow",
"dae" : "does anyone else",
"dbmib" : "do not bother me i am busy",
"diy" : "do it yourself",
"dm" : "direct message",
"dwh" : "during work hours",
"e123" : "easy as one two three",
"eet" : "eastern european time",
"eg" : "example",
"embm" : "early morning business meeting",
"encl" : "enclosed",
"encl." : "enclosed",
"etc" : "and so on",
"faq" : "frequently asked questions",
"fawc" : "for anyone who cares",
"fb" : "facebook",
"fc" : "fingers crossed",
"fig" : "figure",
"fimh" : "forever in my heart",
"ft." : "feet",
"ft" : "featuring",
"ftl" : "for the loss",
"ftw" : "for the win",
"fwiw" : "for what it is worth",
"fyi" : "for your information",
"g9" : "genius",
"gahoy" : "get a hold of yourself",
"gal" : "get a life",
"gcse" : "general certificate of secondary education",
"gfn" : "gone for now",
"gg" : "good game",
"gl" : "good luck",
"glhf" : "good luck have fun",
"gmt" : "greenwich mean time",
"gmta" : "great minds think alike",
"gn" : "good night",
"g.o.a.t" : "greatest of all time",
"goat" : "greatest of all time",
"goi" : "get over it",
"gps" : "global positioning system",
"gr8" : "great",
"gratz" : "congratulations",
"gyal" : "girl",
"h&c" : "hot and cold",
"hp" : "horsepower",
"hr" : "hour",
"hrh" : "his royal highness",
"ht" : "height",
"ibrb" : "i will be right back",
"ic" : "i see",
"icq" : "i seek you",
"icymi" : "in case you missed it",
"idc" : "i do not care",
"idgadf" : "i do not give a damn fuck",
"idgaf" : "i do not give a fuck",
"idk" : "i do not know",
"ie" : "that is",
"i.e" : "that is",
"ifyp" : "i feel your pain",
"IG" : "instagram",
"iirc" : "if i remember correctly",
"ilu" : "i love you",
"ily" : "i love you",
"imho" : "in my humble opinion",
"imo" : "in my opinion",
"imu" : "i miss you",
"iow" : "in other words",
"irl" : "in real life",
"j4f" : "just for fun",
"jic" : "just in case",
"jk" : "just kidding",
"jsyk" : "just so you know",
"l8r" : "later",
"lb" : "pound",
"lbs" : "pounds",
"ldr" : "long distance relationship",
"lmao" : "laugh my ass off",
"lmfao" : "laugh my fucking ass off",
"lol" : "laughing out loud",
"ltd" : "limited",
"ltns" : "long time no see",
"m8" : "mate",
"mf" : "motherfucker",
"mfs" : "motherfuckers",
"mfw" : "my face when",
"mofo" : "motherfucker",
"mph" : "miles per hour",
"mr" : "mister",
"mrw" : "my reaction when",
"ms" : "miss",
"mte" : "my thoughts exactly",
"nagi" : "not a good idea",
"nbc" : "national broadcasting company",
"nbd" : "not big deal",
"nfs" : "not for sale",
"ngl" : "not going to lie",
"nhs" : "national health service",
"nrn" : "no reply necessary",
"nsfl" : "not safe for life",
"nsfw" : "not safe for work",
"nth" : "nice to have",
"nvr" : "never",
"nyc" : "new york city",
"oc" : "original content",
"og" : "original",
"ohp" : "overhead projector",
"oic" : "oh i see",
"omdb" : "over my dead body",
"omg" : "oh my god",
"omw" : "on my way",
"p.a" : "per annum",
"p.m" : "after midday",
"pm" : "prime minister",
"poc" : "people of color",
"pov" : "point of view",
"pp" : "pages",
"ppl" : "people",
"prw" : "parents are watching",
"ps" : "postscript",
"pt" : "point",
"ptb" : "please text back",
"pto" : "please turn over",
"qpsa" : "what happens",
"ratchet" : "rude",
"rbtl" : "read between the lines",
"rlrt" : "real life retweet",
"rofl" : "rolling on the floor laughing",
"roflol" : "rolling on the floor laughing out loud",
"rotflmao" : "rolling on the floor laughing my ass off",
"rt" : "retweet",
"ruok" : "are you ok",
"sfw" : "safe for work",
"sk8" : "skate",
"smh" : "shake my head",
"sq" : "square",
"srsly" : "seriously",
"ssdd" : "same stuff different day",
"tbh" : "to be honest",
"tbs" : "tablespooful",
"tbsp" : "tablespooful",
"tfw" : "that feeling when",
"thks" : "thank you",
"tho" : "though",
"thx" : "thank you",
"tia" : "thanks in advance",
"til" : "today i learned",
"tl;dr" : "too long i did not read",
"tldr" : "too long i did not read",
"tmb" : "tweet me back",
"tntl" : "trying not to laugh",
"ttyl" : "talk to you later",
"u" : "you",
"u2" : "you too",
"u4e" : "yours for ever",
"utc" : "coordinated universal time",
"w/" : "with",
"w/o" : "without",
"w8" : "wait",
"wassup" : "what is up",
"wb" : "welcome back",
"wtf" : "what the fuck",
"wtg" : "way to go",
"wtpa" : "where the party at",
"wuf" : "where are you from",
"wuzup" : "what is up",
"wywh" : "wish you were here",
"yd" : "yard",
"ygtr" : "you got that right",
"ynk" : "you never know",
"zzz" : "sleeping bored and tired"
}
def convert_abbrev(word):
return abbreviations[word] if word in abbreviations.keys() else word<categorify>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam)
def convert_abbrev_in_text(text):
tokens = word_tokenize(text)
 tokens = [convert_abbrev(word)for word in tokens]
text = ' '.join(tokens)
 return text
df_all['text']=df_all['text'].apply(lambda x : convert_abbrev_in_text(x))<feature_engineering>
all_data = pd.get_dummies(all_data)
 print(all_data.shape )
df_all['text']=df_all['text'].apply(lambda x : word_tokenize(x))<feature_engineering>
train = all_data[:ntrain]
test = all_data[ntrain:]
stop = set(stopwords.words('english'))
def remove_stopwords(text):
words = [w for w in text if w not in stop]
return ' '.join(words)
 df_all['text']=df_all['text'].apply(lambda x : remove_stopwords(x))<define_variables>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
cl_ch_len = df_all['text'].apply(lambda x: len(x))
cl_wd_len = df_all['text'].str.split().map(lambda x: len(x))
print('Max words length for cleaned tweets: {}'.format(max(cl_wd_len)))
 print('Max characters length for cleaned tweets: {}'.format(max(cl_ch_len)))
 MAX_LEN = max(cl_wd_len )<remove_duplicates>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
def create_corpus(df):
corpus=[]
for tweet in tqdm(df_all['text']):
words=[word.lower() for word in word_tokenize(tweet)if(( word.isalpha() ==1)&(word not in stop)) ]
corpus.append(words)
 return corpus
corpus = create_corpus(df_all)
 corpus[0]<load_from_csv>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
glove_embedding_dict={}
with open('.. /input/glove-global-vectors-for-word-representation/glove.6B.200d.txt','r')as f:
for line in tqdm(f):
values=line.split()
word=values[0]
vectors=np.asarray(values[1:],'float32')
 glove_embedding_dict[word]=vectors<define_variables>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
W_E_DIM = 200<categorify>
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5 )
tokenizer_obj=Tokenizer()
tokenizer_obj.fit_on_texts(corpus)
 sequences=tokenizer_obj.texts_to_sequences(corpus)
 tweet_pad=pad_sequences(sequences,maxlen=MAX_LEN,truncating='post',padding='post' )<count_unique_values>
GBoost = GradientBoostingRegressor(n_estimators=200, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )
word_index=tokenizer_obj.word_index
print('Number of unique words:',len(word_index))<define_variables>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
nthread = -1)
num_words=len(word_index)+1
embedding_matrix=np.zeros(( num_words,W_E_DIM))
for word,i in tqdm(word_index.items()):
if i < num_words:
emb_vec=glove_embedding_dict.get(word)
 if emb_vec is not None:
embedding_matrix[i]=emb_vec<split>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )
train_text = tweet_pad[:train.shape[0]]
test_text = tweet_pad[train.shape[0]:]
X_train,X_dev,Y_train,Y_dev=train_test_split(train_text,train['target'].values,test_size=0.2)
 print('Shape of train',X_train.shape)
 print("Shape of Validation ",X_dev.shape )<choose_model_class>
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
model=Sequential()
embedding=Embedding(num_words,W_E_DIM,embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_LEN,trainable=False)
 model.add(embedding)
 model.add(SpatialDropout1D(0.2))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
optimzer=Adam(learning_rate=3e-4)
 model.compile(loss='binary_crossentropy',optimizer=optimzer,metrics=['accuracy'])
 model.summary()<train_model>
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
history=model.fit(X_train,Y_train,batch_size=4,
epochs=10,validation_data=(X_dev,Y_dev),verbose=2)
<predict_on_test>
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
pred = model.predict(test_text)
 pred = pred.round().astype('int' )<save_to_csv>
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
df_sub = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 df_sub['target'] = pred
df_sub = df_sub[['id', 'target']]
df_sub.to_csv('lstm_submission.csv', index=False, header=True)
 df_sub.head(10 )<choose_model_class>
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
%%time
bert_url = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/2'
bert_layer = hub.KerasLayer(bert_url, trainable=True, name='Bert_layer' )<categorify>
score = rmsle_cv(model_lgb)
 print("lightfm score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def bert_encode(texts, tokenizer, max_len=512):
input_tokens = []
input_masks = []
input_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
input_tokens.append(tokens)
 input_masks.append(pad_masks)
 input_segments.append(segment_ids)
 return np.array(input_tokens), np.array(input_masks), np.array(input_segments )<choose_model_class>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 cls_output = sequence_output[:,0,:]
output = Dense(1, activation='sigmoid' )(cls_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=output)
 model.compile(Adam(lr=1e-5), loss = 'binary_crossentropy',metrics=['accuracy'])
 return model 
<feature_engineering>
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
train_text = df_all[:train.shape[0]].text
test_text = df_all[train.shape[0]:].text
train_input = bert_encode(train_text, tokenizer, max_len=160)
 test_input = bert_encode(test_text, tokenizer, max_len = 160)
 train_labels = train.target.values<train_model>
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=5,
callbacks=[checkpoint],
batch_size=16
)<predict_on_test>
averaged_models.fit(train.values, y_train)
 y_pre_averaged_models = averaged_models.predict(test.values)
model.load_weights('model.h5')
 test_pred = model.predict(test_input )<save_to_csv>
stacked_averaged_models.fit(train.values, y_train)
 y_pre_stacked_averaged_models = stacked_averaged_models.predict(test.values)
submission['target'] = test_pred.round().astype(int)
 submission.to_csv('bert_submission.csv', index=False )<load_from_csv>
model_xgb.fit(train.values, y_train)
 xgb_pred = model_xgb.predict(test.values)
test_set = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 train_set = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv" )<import_modules>
model_lgb.fit(train.values, y_train)
 lgb_pred = model_lgb.predict(test.values)
import seaborn as sns
from matplotlib import pyplot as plt<count_values>
ensemble = y_pre_averaged_models*0.5 + y_pre_stacked_averaged_models *0.5
print("train set")
 print(train_set.count())
 print("test set")
 print(test_set.count() )<install_modules>
pre_df = pd.DataFrame()
pre_df["SalePrice"] = np.exp(ensemble)-1
pre_df = pre_df[["SalePrice"]]
!pip install emoji<string_transform>
submission = pd.DataFrame()
submission['Id'] = test_ID
submission = pd.concat([submission['Id'],pre_df["SalePrice"]],axis=1)
 submission
<feature_engineering><EOS> 
submission.to_csv('1.csv',index=False )