Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<categorify>
np.set_printoptions(suppress=True)
 pio.templates.default = "plotly_white"
indices=tokenizer.batch_encode_plus(texts,max_length=64,add_special_tokens=True, return_attention_mask=True,pad_to_max_length=True,truncation=True)
 input_ids=indices["input_ids"]
attention_masks=indices["attention_mask"]<split>
train_data=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test_data = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 pd.set_option('display.max_columns', 500)
 print(train_data.shape)
 train_data.describe()
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels,
random_state=42, test_size=0.2)
 train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,
random_state=42, test_size=0.2 )<data_type_conversions>
X_train = train_data.loc[:,'MSSubClass':'SaleCondition']
y_train = train_data.SalePrice
X_test = test_data.loc[:,'MSSubClass':'SaleCondition']
train_inputs = torch.tensor(train_inputs)
 validation_inputs = torch.tensor(validation_inputs)
 train_labels = torch.tensor(train_labels, dtype=torch.long)
 validation_labels = torch.tensor(validation_labels, dtype=torch.long)
 train_masks = torch.tensor(train_masks, dtype=torch.long)
 validation_masks = torch.tensor(validation_masks, dtype=torch.long )<load_pretrained>
def missing_ratio(all_data):
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 return missing_data
batch_size = 32
train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = SequentialSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size )<compute_test_metric>
train_na=missing_ratio(X_train)
 test_na=missing_ratio(X_test)
 compare_na = pd.concat([train_na, test_na], axis=1)
 compare_na.columns=['Train Missing Ratio','Test Missing Ratio']
compare_na.fillna(0 )
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )<categorify>
drop_col=list(compare_na[(compare_na['Train Missing Ratio'] > 99)|(compare_na['Test Missing Ratio'] >99)].index)
 drop_col
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))<define_variables>
def manu_miss(all_data):
for NA_to_0 in('MasVnrArea' ,'GarageYrBlt', 'GarageArea', 'GarageCars','BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[NA_to_0] = all_data[NA_to_0].fillna(0)
 for NA_to_None in('MiscFeature','Alley','Fence','FireplaceQu','GarageType', 'GarageFinish','GarageQual', 'GarageCond','BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2','MasVnrType'):
all_data[NA_to_None] = all_data[NA_to_None].fillna('None')
 all_data['Functional'] = all_data['Functional'].fillna('Typ')
 all_data['LotFrontage']=all_data.groupby('Neighborhood')['LotFrontage'].transform(lambda x:x.fillna(x.median()))
all_data['MSZoning']=all_data.groupby('Neighborhood')['MSZoning'].transform(lambda x:x.fillna(x.mode() [0]))
all_data['Electrical']=all_data.groupby('YearRemodAdd')['Electrical'].transform(lambda x:x.fillna(x.mode() [0]))
all_data['KitchenQual']=all_data.groupby('ExterQual')['KitchenQual'].transform(lambda x:x.fillna(x.mode() [0]))
all_data['SaleType']=all_data.groupby('SaleCondition')['SaleType'].transform(lambda x:x.fillna(x.mode() [0]))
return all_data
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 loss_values = []
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 50 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_loss / len(train_dataloader)
 loss_values.append(avg_train_loss)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epoch took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Training complete!" )<train_model>
X_train=manu_miss(X_train)
 X_test=manu_miss(X_test )
print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
preds=[]
true=[]
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in validation_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad() :
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
preds.append(logits)
 true.append(label_ids)
 tmp_eval_accuracy = flat_accuracy(logits, label_ids)
 eval_accuracy += tmp_eval_accuracy
nb_eval_steps += 1
print(" Accuracy: {0:.2f}".format(eval_accuracy/nb_eval_steps))
print(" Validation took: {:}".format(format_time(time.time() - t0)) )<define_variables>
X_train = pd.concat([imputed_X_train_num, imputed_X_train_cat], axis=1)
 X_test = pd.concat([imputed_X_test_num, imputed_X_test_cat], axis=1)
 X_train.shape
flat_predictions = [item for sublist in preds for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
flat_true_labels = [item for sublist in true for item in sublist]<compute_test_metric>
def ordered_feature_transform(all_data):
QualMap={'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'None':0}
Qualmap_features=('ExterQual','ExterCond','BsmtQual','BsmtCond','HeatingQC','KitchenQual','FireplaceQu','GarageQual','GarageCond')
 for map_feature in Qualmap_features:
all_data[map_feature]=all_data[map_feature].map(QualMap)
 BsmtExposureMap={'Gd':4,'Av':3,'Mn':2,'No':1,'None':0}
all_data['BsmtExposure']=all_data['BsmtExposure'].map(BsmtExposureMap)
 BsmtFinTypeMap={'GLQ':6,'ALQ':5,'BLQ':4,'Rec':3,'LwQ':2,'Unf':1,'None':0}
BsmtFinTypemap_features=('BsmtFinType1','BsmtFinType2')
 for map_feature in BsmtFinTypemap_features:
all_data[map_feature]=all_data[map_feature].map(BsmtFinTypeMap)
 UtilitiesMap={'AllPub':4,'NoSewr':3,'NoSeWa':2,'ELO':1,'None':0}
all_data['Utilities']=all_data['Utilities'].map(UtilitiesMap)
 GarageFinishMap={'Fin':3,'RFn':2,'Unf':1,'None':0}
all_data['GarageFinish']=all_data['GarageFinish'].map(GarageFinishMap)
 CentralAirMap={'Y':1,'N':0}
all_data['CentralAir']=all_data['CentralAir'].map(CentralAirMap)
 MSSubClassMap={'180':1, '30':2, '45':2,'190':3, '50':3, '90':3,'85':4, '40':4, '160':4,'70':5, '20':5, '75':5, '80':5, '150':5,'120': 6, '60':6}
all_data['MSSubClass'] = all_data['MSSubClass'].astype(int ).apply(str)
 all_data['MSSubClass']=all_data['MSSubClass'].map(MSSubClassMap)
 MSZoningMap={'C(all)':1, 'RH':2, 'RM':2, 'RL':3, 'FV':4}
all_data['MSZoning']=all_data['MSZoning'].map(MSZoningMap)
 NeighborhoodMap={'MeadowV':1,'IDOTRR':2, 'BrDale':2,'OldTown':3, 'Edwards':3, 'BrkSide':3,'Sawyer':4, 'Blueste':4, 'SWISU':4, 'NAmes':4,
'NPkVill':5, 'Mitchel':5,'SawyerW':6, 'Gilbert':6, 'NWAmes':6,'Blmngtn':7, 'CollgCr':7, 'ClearCr':7, 'Crawfor':7,
'Veenker':8, 'Somerst':8, 'Timber':8,'StoneBr':9,'NoRidge':10, 'NridgHt':10}
all_data['Neighborhood']=all_data['Neighborhood'].map(NeighborhoodMap)
 HouseStyleMap={'1.5Unf':1,'1.5Fin':2, '2.5Unf':2, 'SFoyer':2, '1Story':3, 'SLvl':3,'2Story':4, '2.5Fin':4}
all_data['HouseStyle']=all_data['HouseStyle'].map(HouseStyleMap)
 MasVnrTypeMap={'BrkCmn':1, 'None':1, 'BrkFace':2, 'Stone':3}
all_data['MasVnrType']=all_data['MasVnrType'].map(MasVnrTypeMap)
 FoundationMap={'Slab':1, 'BrkTil':2, 'CBlock':2, 'Stone':2,'Wood':3, 'PConc':4}
all_data['Foundation']=all_data['Foundation'].map(FoundationMap)
 GarageTypeMap={'CarPort':1, 'None':1,'Detchd':2,'2Types':3, 'Basment':3,'Attchd':4, 'BuiltIn':5}
all_data['GarageType']=all_data['GarageType'].map(GarageTypeMap)
 SaleConditionMap={'AdjLand':1, 'Abnorml':2, 'Alloca':2, 'Family':2, 'Normal':3, 'Partial':4}
all_data['SaleCondition']=all_data['SaleCondition'].map(SaleConditionMap)
 return all_data
print(classification_report(flat_predictions,flat_true_labels))<define_variables>
X_train=ordered_feature_transform(X_train)
 X_test=ordered_feature_transform(X_test )
comments1 = df_test.text.values
indices1=tokenizer.batch_encode_plus(comments1,max_length=128,add_special_tokens=True, return_attention_mask=True,pad_to_max_length=True,truncation=True)
 input_ids1=indices1["input_ids"]
attention_masks1=indices1["attention_mask"]
prediction_inputs1= torch.tensor(input_ids1)
 prediction_masks1 = torch.tensor(attention_masks1)
 batch_size = 32
prediction_data1 = TensorDataset(prediction_inputs1, prediction_masks1)
 prediction_sampler1 = SequentialSampler(prediction_data1)
 prediction_dataloader1 = DataLoader(prediction_data1, sampler=prediction_sampler1, batch_size=batch_size)
<predict_on_test>
def add_new_feature(all_data):
all_data['BsmtFin'] =all_data['BsmtFinType1']*all_data['BsmtFinSF1']+all_data['BsmtFinType2']*all_data['BsmtFinSF2']
all_data['haspool'] = all_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['has2ndfloor'] = all_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasgarage'] = all_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasbsmt'] = all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasfireplace'] = all_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 all_data['LargeHouseLevel'] = all_data['GrLivArea'].apply(lambda x: 1 if x > 2750 else 0)
 all_data['OldHouse'] = all_data['YearBuilt'].apply(lambda x: 1 if x < 1910 else 0)
 all_data['BsmtUnfSF_cat'] = all_data['BsmtUnfSF'].apply(lambda x: 1 if x <=500 else 2 if x <= 1000 else 3 if x <= 1500 else 4 if x <= 2000 else 5)
 all_data['Qual']=all_data['BsmtQual']+all_data['HeatingQC']+all_data['KitchenQual']+all_data['FireplaceQu']+all_data['GarageQual']
all_data["AllSF"] = all_data["GrLivArea"] + all_data["TotalBsmtSF"]
all_data['Total_porch'] = all_data['OpenPorchSF']+ all_data['3SsnPorch'] + all_data['EnclosedPorch'] + all_data['ScreenPorch'] + all_data['WoodDeckSF']
all_data['YrBltAndRemod'] = all_data['YearBuilt'] + all_data['YearRemodAdd']
all_data['Total_Bathrooms'] = all_data['FullBath'] +(0.5 * all_data['HalfBath'])+ all_data['BsmtFullBath'] +(0.5 * all_data['BsmtHalfBath'])
 all_data['Total_sqr_footage'] = all_data['BsmtFinSF1'] + all_data['BsmtFinSF2'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
return all_data
print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs1)))
 model.eval()
predictions = []
for batch in prediction_dataloader1:
batch = tuple(t.to(device)for t in batch)
 b_input_ids1, b_input_mask1 = batch
with torch.no_grad() :
outputs1 = model(b_input_ids1, token_type_ids=None,
attention_mask=b_input_mask1)
 logits1 = outputs1[0]
logits1 = logits1.detach().cpu().numpy()
predictions.append(logits1)
 flat_predictions = [item for sublist in predictions for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()<save_to_csv>
X_train=add_new_feature(X_train)
 X_test=add_new_feature(X_test )
sample_sub=pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 submit=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':flat_predictions})
<data_type_conversions>
def drop_rest(all_data):
all_data=all_data.drop(['BsmtFinType1','BsmtFinSF1','BsmtFinType2','BsmtFinSF2','MiscFeature','1stFlrSF','2ndFlrSF','LowQualFinSF'], axis=1)
 return all_data
df_leak = pd.read_csv('/kaggle/input/disasters-on-social-media/socialmedia-disaster-tweets-DFE.csv', encoding ='ISO-8859-1')[['choose_one', 'text']]
df_leak['target'] =(df_leak['choose_one'] == 'Relevant' ).astype(np.int8)
 df_leak['id'] = df_leak.index.astype(np.int16)
 df_leak.drop(columns=['choose_one', 'text'], inplace=True)
 df_test = df_test.merge(df_leak, on=['id'], how='left')
 print('Leaked Data Set Shape = {}'.format(df_leak.shape))
print('Leaked Data Set Memory Usage = {:.2f} MB'.format(df_leak.memory_usage().sum() / 1024**2))
perfect_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 perfect_submission['target'] = df_test['target'].values
perfect_submission.to_csv('submission.csv', index=False)
 perfect_submission.describe()<import_modules>
X_train=drop_rest(X_train)
 X_test=drop_rest(X_test )
import pandas as pd
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import transformers as trfo
import sklearn.model_selection as ms
import sklearn.metrics as m
from functools import partial
import hyperopt as ho
import pickle
import re
import string
import itertools
import time
import tqdm
import operator
import xgboost<load_from_csv>
def manu_dum(all_data):
for new_col in('Artery','Feedr','Norm','RRNn','RRAn','PosN','PosA','RRNe','RRAe'):
all_data[new_col]=0.0
for col in('Condition1','Condition2'):
all_data.loc[all_data[col] == new_col, new_col] = 1.0
all_data=all_data.drop(['Condition1','Condition2'], axis=1)
 for new_col in('AsbShng','AsphShn','BrkComm','BrkFace','CBlock','CemntBd','HdBoard','ImStucc','MetalSd','Other','Plywood','Stone','Stucco','VinylSd','Wd Shng','WdShing'):
all_data[new_col]=0.0
for col in('Exterior1st','Exterior2nd'):
all_data.loc[all_data[col] == new_col, new_col] = 1.0
all_data=all_data.drop(['Exterior1st','Exterior2nd'], axis=1)
 return all_data
train_df = pd.read_csv('.. /input/nlp-getting-started/train.csv' )<load_pretrained>
X_train=manu_dum(X_train)
 X_test=manu_dum(X_test )
tokenizer = trfo.BertTokenizer.from_pretrained('bert-large-uncased' )<define_variables>
ObjectData=X_train.dtypes[X_train.dtypes == "object"].index
print(X_train[ObjectData].columns)
 cat_features=list(X_train[ObjectData].columns )
tokenizer.encode('London!' )<categorify>
cat_features=['Street', 'Alley', 'LotShape', 'LandContour', 'LotConfig', 'LandSlope',
'RoofStyle', 'RoofMatl', 'Heating', 'Electrical',
'Functional', 'PavedDrive', 'Fence', 'SaleType']
tokenizer.decode(101), tokenizer.decode(2414), tokenizer.decode(999), tokenizer.decode(102 )<feature_engineering>
for cat_feature in cat_features:
encoder = LabelEncoder()
encoder.fit(list(X_train[cat_feature].values))
X_train[cat_feature] = encoder.transform(list(X_train[cat_feature].values))
X_test[cat_feature] = encoder.transform(list(X_test[cat_feature].values))
def build_vocab(sentences):
vocab = {}
for sentence in tqdm.tqdm(sentences):
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab<categorify>
X_train = pd.get_dummies(X_train)
 X_test = pd.get_dummies(X_test)
 X_train, X_test = X_train.align(X_test, join='left', axis=1, fill_value=0)
 print(X_train.shape,X_test.shape )
def check_coverage(vocab, embeddings_index):
a = {}
oov = {}
k = 0
i = 0
for word in tqdm.tqdm(vocab):
try:
a[word] = embeddings_index[word]
k += vocab[word]
except:
oov[word] = vocab[word]
i += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(a)/ len(vocab)))
 print('Found embeddings for {:.2%} of full text'.format(k /(k + i)) )<string_transform>
numeric_feats = X_train.dtypes[X_train.dtypes != "object"].index
skewed_feats = X_train[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
 skewness = pd.DataFrame({'Skew' :skewed_feats})
 print(skewness.head(30))
X_train[skewness.index] = np.log1p(X_train[skewness.index])
 X_test[skewness.index] = np.log1p(X_test[skewness.index])
 y_train= np.log1p(y_train )
vocab = build_vocab(train_df['text'].apply(lambda x: x.split() ).values)
 check_coverage(vocab, tokenizer.get_vocab() )<feature_engineering>
iso = IsolationForest(max_samples=50, random_state=52,contamination=0.005 ).fit(X_train )
train_df['text'] = train_df['text'].apply(lambda x: x.lower())
 vocab = build_vocab(train_df['text'].apply(lambda x: x.split() ).values)
 check_coverage(vocab, tokenizer.get_vocab() )<define_variables>
X_train["Outlier"]=iso.predict(X_train)
 X_test["Outlier"]=iso.predict(X_test )
def remove_url(text):
return re.sub(r'https?:\/\/t.co\/[A-Za-z0-9]+', '', text )<drop_column>
print(X_train["Outlier"].value_counts())
 print(X_test["Outlier"].value_counts() )
def remove_user(text):
text = re.sub(r'\@[A-Za-z0-9]+', '', text)
 return text<feature_engineering>
model_select = SelectFromModel(Lasso(alpha=0.0005,tol=0.001))
ms = model_select.fit(X_train,y_train)
 selected_feat = X_train.columns[(ms.get_support())]
print('total features: {}'.format(( X_train.shape[1])))
 print('selected features: {}'.format(len(selected_feat)))
 print('features with coefficients shrank to zero: {}'.format(
 np.sum(ms.estimator_.coef_ == 0)))
 shrink_zero_num=np.sum(ms.estimator_.coef_ == 0)
 print("Number of features which coefficient was shrank to zero: ",shrink_zero_num)
 removed_feats = X_train.columns[(ms.estimator_.coef_ == 0 ).ravel().tolist() ]
print("The removed features: ",removed_feats )
train_df['text'] = train_df['text'].apply(lambda x: remove_url(x))
train_df['text'] = train_df['text'].apply(lambda x: remove_user(x))
vocab = build_vocab(train_df['text'].apply(lambda x: x.split() ).values)
 check_coverage(vocab, tokenizer.get_vocab() )<define_variables>
rf = RandomForestRegressor(n_estimators=200)
 rf = rf.fit(X_train,y_train)
 rf_importance=pd.DataFrame({'Features':np.array(X_train.columns),'Importance':rf.feature_importances_*100 } ).sort_values(by='Importance', ascending=False)
 rf_importance
abbreviations_mapping = {
"$" : " dollar ",
"€" : " euro ",
"4ao" : "for adults only",
"a.m" : "before midday",
"a3" : "anytime anywhere anyplace",
"aamof" : "as a matter of fact",
"acct" : "account",
"adih" : "another day in hell",
"afaic" : "as far as i am concerned",
"afaict" : "as far as i can tell",
"afaik" : "as far as i know",
"afair" : "as far as i remember",
"afk" : "away from keyboard",
"app" : "application",
"approx" : "approximately",
"apps" : "applications",
"asap" : "as soon as possible",
"asl" : "age, sex, location",
"atk" : "at the keyboard",
"ave." : "avenue",
"aymm" : "are you my mother",
"ayor" : "at your own risk",
"b&b" : "bed and breakfast",
"b+b" : "bed and breakfast",
"b.c" : "before christ",
"b2b" : "business to business",
"b2c" : "business to customer",
"b4" : "before",
"b4n" : "bye for now",
"b@u" : "back at you",
"bae" : "before anyone else",
"bak" : "back at keyboard",
"bbbg" : "bye bye be good",
"bbc" : "british broadcasting corporation",
"bbias" : "be back in a second",
"bbl" : "be back later",
"bbs" : "be back soon",
"be4" : "before",
"bfn" : "bye for now",
"blvd" : "boulevard",
"bout" : "about",
"brb" : "be right back",
"bros" : "brothers",
"brt" : "be right there",
"bsaaw" : "big smile and a wink",
"btw" : "by the way",
"bwl" : "bursting with laughter",
"c/o" : "care of",
"cet" : "central european time",
"cf" : "compare",
"cia" : "central intelligence agency",
"csl" : "can not stop laughing",
"cu" : "see you",
"cul8r" : "see you later",
"cv" : "curriculum vitae",
"cwot" : "complete waste of time",
"cya" : "see you",
"cyt" : "see you tomorrow",
"dae" : "does anyone else",
"dbmib" : "do not bother me i am busy",
"diy" : "do it yourself",
"dm" : "direct message",
"dwh" : "during work hours",
"e123" : "easy as one two three",
"eet" : "eastern european time",
"eg" : "example",
"embm" : "early morning business meeting",
"encl" : "enclosed",
"encl." : "enclosed",
"etc" : "and so on",
"faq" : "frequently asked questions",
"fawc" : "for anyone who cares",
"fb" : "facebook",
"fc" : "fingers crossed",
"fig" : "figure",
"fimh" : "forever in my heart",
"ft." : "feet",
"ft" : "featuring",
"ftl" : "for the loss",
"ftw" : "for the win",
"fwiw" : "for what it is worth",
"fyi" : "for your information",
"g9" : "genius",
"gahoy" : "get a hold of yourself",
"gal" : "get a life",
"gcse" : "general certificate of secondary education",
"gfn" : "gone for now",
"gg" : "good game",
"gl" : "good luck",
"glhf" : "good luck have fun",
"gmt" : "greenwich mean time",
"gmta" : "great minds think alike",
"gn" : "good night",
"g.o.a.t" : "greatest of all time",
"goat" : "greatest of all time",
"goi" : "get over it",
"gps" : "global positioning system",
"gr8" : "great",
"gratz" : "congratulations",
"gyal" : "girl",
"h&c" : "hot and cold",
"hp" : "horsepower",
"hr" : "hour",
"hrh" : "his royal highness",
"ht" : "height",
"ibrb" : "i will be right back",
"ic" : "i see",
"icq" : "i seek you",
"icymi" : "in case you missed it",
"idc" : "i do not care",
"idgadf" : "i do not give a damn fuck",
"idgaf" : "i do not give a fuck",
"idk" : "i do not know",
"ie" : "that is",
"i.e" : "that is",
"ifyp" : "i feel your pain",
"IG" : "instagram",
"iirc" : "if i remember correctly",
"ilu" : "i love you",
"ily" : "i love you",
"imho" : "in my humble opinion",
"imo" : "in my opinion",
"imu" : "i miss you",
"iow" : "in other words",
"irl" : "in real life",
"j4f" : "just for fun",
"jic" : "just in case",
"jk" : "just kidding",
"jsyk" : "just so you know",
"l8r" : "later",
"lb" : "pound",
"lbs" : "pounds",
"ldr" : "long distance relationship",
"lmao" : "laugh my ass off",
"lmfao" : "laugh my fucking ass off",
"lol" : "laughing out loud",
"ltd" : "limited",
"ltns" : "long time no see",
"m8" : "mate",
"mf" : "motherfucker",
"mfs" : "motherfuckers",
"mfw" : "my face when",
"mofo" : "motherfucker",
"mph" : "miles per hour",
"mr" : "mister",
"mrw" : "my reaction when",
"ms" : "miss",
"mte" : "my thoughts exactly",
"nagi" : "not a good idea",
"nbc" : "national broadcasting company",
"nbd" : "not big deal",
"nfs" : "not for sale",
"ngl" : "not going to lie",
"nhs" : "national health service",
"nrn" : "no reply necessary",
"nsfl" : "not safe for life",
"nsfw" : "not safe for work",
"nth" : "nice to have",
"nvr" : "never",
"nyc" : "new york city",
"oc" : "original content",
"og" : "original",
"ohp" : "overhead projector",
"oic" : "oh i see",
"omdb" : "over my dead body",
"omg" : "oh my god",
"omw" : "on my way",
"p.a" : "per annum",
"p.m" : "after midday",
"pm" : "prime minister",
"poc" : "people of color",
"pov" : "point of view",
"pp" : "pages",
"ppl" : "people",
"prw" : "parents are watching",
"ps" : "postscript",
"pt" : "point",
"ptb" : "please text back",
"pto" : "please turn over",
"qpsa" : "what happens",
"ratchet" : "rude",
"rbtl" : "read between the lines",
"rlrt" : "real life retweet",
"rofl" : "rolling on the floor laughing",
"roflol" : "rolling on the floor laughing out loud",
"rotflmao" : "rolling on the floor laughing my ass off",
"rt" : "retweet",
"ruok" : "are you ok",
"sfw" : "safe for work",
"sk8" : "skate",
"smh" : "shake my head",
"sq" : "square",
"srsly" : "seriously",
"ssdd" : "same stuff different day",
"tbh" : "to be honest",
"tbs" : "tablespooful",
"tbsp" : "tablespooful",
"tfw" : "that feeling when",
"thks" : "thank you",
"tho" : "though",
"thx" : "thank you",
"tia" : "thanks in advance",
"til" : "today i learned",
"tl;dr" : "too long i did not read",
"tldr" : "too long i did not read",
"tmb" : "tweet me back",
"tntl" : "trying not to laugh",
"ttyl" : "talk to you later",
"u" : "you",
"u2" : "you too",
"u4e" : "yours for ever",
"utc" : "coordinated universal time",
"w/" : "with",
"w/o" : "without",
"w8" : "wait",
"wassup" : "what is up",
"wb" : "welcome back",
"wtf" : "what the fuck",
"wtg" : "way to go",
"wtpa" : "where the party at",
"wuf" : "where are you from",
"wuzup" : "what is up",
"wywh" : "wish you were here",
"yd" : "yard",
"ygtr" : "you got that right",
"ynk" : "you never know",
"zzz" : "sleeping bored and tired"
}<define_variables>
model = SelectFromModel(rf, prefit=True,threshold=0.0005)
 columns_select_rf=X_train.columns[model.get_support() ]
print("kept {} features after tree-based feature selection".format(len(columns_select_rf)) ,"
The features are : ",columns_select_rf )
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not",
"don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did",
"how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am",
"I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would",
"it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have",
"mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not",
"needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not",
"shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have",
"shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have",
"that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have",
"they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would",
"we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will",
"what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did",
"where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have",
"will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all",
"y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have",
"you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have"
} 
<define_variables>
def select_features(X_train, y_train, X_test):
fs = SelectKBest(score_func=f_regression, k='all')
 fs.fit(X_train, y_train)
 X_train_fs = fs.transform(X_train)
 X_test_fs = fs.transform(X_test)
 return X_train_fs, X_test_fs, fs
X_train_fs, X_test_fs, fs = select_features(X_train, y_train, X_test)
 clf_importance=pd.DataFrame({'Features':np.array(X_train.columns),'Importance':fs.scores_ } ).sort_values(by='Importance', ascending=False)
 clf_importance
synonyms_mapping = {
"&amp;": "&",
"retweet": "response to",
"wildfire": "flame",
"reddit": "social network",
"legionnaires": "disease",
"thunderstorm": "storm",
"sinkhole": "crater",
"derailment": "runs off its rails",
"windstorm": "storm",
"twister": "tornado",
"rescuers": "people who rescue",
"whirlwind": "hurricane",
"sandstorm": "storm",
"mudslide": "avalanche",
"curfew": "time when individuals are required to return to their houses",
"armageddon": "disaster"
}<string_transform>
def select_features(X_train, y_train, X_test):
fs = SelectKBest(score_func=mutual_info_regression, k='all')
 fs.fit(X_train, y_train)
 X_train_fs = fs.transform(X_train)
 X_test_fs = fs.transform(X_test)
 return X_train_fs, X_test_fs, fs
X_train_fs, X_test_fs, fs = select_features(X_train, y_train, X_test)
 clf_importance=pd.DataFrame({'Features':np.array(X_train.columns),'Importance':fs.scores_ } ).sort_values(by='Importance', ascending=False)
 clf_importance
def translate_with_mapping(text, dictionary):
text = ' '.join([dictionary[t] if t in dictionary else t for t in text.split(' ')])
 return text<feature_engineering>
X_train=X_train[columns_select_mul]
X_test=X_test[columns_select_mul]
train_df['text'] = train_df['text'].apply(lambda x: translate_with_mapping(x, synonyms_mapping))
train_df['text'] = train_df['text'].apply(lambda x: translate_with_mapping(x, abbreviations_mapping))
train_df['text'] = train_df['text'].apply(lambda x: translate_with_mapping(x, contraction_mapping))
vocab = build_vocab(train_df['text'].apply(lambda x: x.split() ).values)
 check_coverage(vocab, tokenizer.get_vocab() )<remove_duplicates>
kmeans = KMeans(n_clusters=3, random_state=25 ).fit(X_train)
 X_train['cluster']=kmeans.labels_
X_test['cluster']=kmeans.predict(X_test )
def remove_punct_dup(text):
punc = set(string.punctuation)
 newtext = []
for k, g in itertools.groupby(text):
if k in punc:
newtext.append(k)
 else:
newtext.extend(g)
 return ''.join(newtext )<feature_engineering>
pd.DataFrame(kmeans.labels_ ).value_counts()
train_df['text'] = train_df['text'].apply(lambda x: remove_punct_dup(x))
vocab = build_vocab(train_df['text'].apply(lambda x: x.split() ).values)
 check_coverage(vocab, tokenizer.get_vocab() )<train_model>
X_train['cluster']=X_train['cluster'].astype(str)
 X_test['cluster']=X_test['cluster'].astype(str)
 X_train = pd.get_dummies(X_train)
 X_test = pd.get_dummies(X_test)
 X_train, X_test = X_train.align(X_test, join='left', axis=1, fill_value=0 )
def init_tpu(tpu):
tf.tpu.experimental.initialize_tpu_system(tpu )<choose_model_class>
X_train=X_train.drop(X_train[to_drop], axis=1)
 X_test=X_test.drop(X_test[to_drop], axis=1 )
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 init_tpu(tpu)
 tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu )<load_pretrained>
X_tra,X_val,y_tra,y_val = train_test_split(X_train, y_train, test_size = 0.25, random_state = 25)
 print(X_tra.shape)
 print(X_val.shape)
 print(y_tra.shape)
 print(y_val.shape )
def load_trials(name, remove_last=True):
trials = pickle.load(open(name, 'rb'))
if remove_last:
trials = remove_last_trial(trials)
 return trials<feature_engineering>
start_time = time.time()
regr = make_pipeline(RobustScaler() ,linear_model.LinearRegression())
 regr.fit(X_tra,y_tra)
 y_tra_regr_pred=regr.predict(X_tra)
 print("The MSE on train set:",np.sqrt(metrics.mean_squared_error(y_tra, y_tra_regr_pred)))
 y_val_regr_pred = regr.predict(X_val)
 print("The MSE on validation set:",np.sqrt(metrics.mean_squared_error(y_val, y_val_regr_pred)))
 regr_mse = cross_val_score(regr , X_train, y_train, scoring='neg_mean_squared_error',cv=5)
 print("MSE on Cross-validation:
",np.sqrt(-regr_mse))
print("The mean MSE on Cross-validation is {:.3f}".format(np.mean(np.sqrt(-regr_mse))))
print("--- %s seconds ---" %(time.time() - start_time))
plt.figure('regr')
 plt.plot([np.expm1(y_val.min()),np.expm1(y_val.max())],[np.expm1(y_val.min()),np.expm1(y_val.max())],'k--',lw=2)
 plt.scatter(np.expm1(y_val),np.expm1(y_val_regr_pred))
plt.xlabel('Fact Price')
 plt.ylabel('Predict Price')
 plt.title('Predict Effect Using Linear Regression')
 plt.show()
def remove_last_trial(old_trials):
trials = ho.Trials()
for trial in old_trials.trials[:-1]:
hyperopt_trial = ho.Trials().new_trial_docs(
 tids=[None],
specs=[None],
results=[None],
miscs=[None])
 hyperopt_trial[0] = trial
trials.insert_trial_docs(hyperopt_trial)
 trials.refresh()
return trials<load_pretrained>
rcv = RidgeCV(alphas=np.arange(3.0,6.0,0.1))
rcv.fit(X_tra,y_tra)
 print(rcv.alpha_)
 y_tra_rcv_pred=rcv.predict(X_tra)
 print("使用Ridge Regression cv模型的均方根误差为(train):",np.sqrt(metrics.mean_squared_error(y_tra, y_tra_rcv_pred)))
 y_rcv_pred = rcv.predict(X_val)
 print("使用Ridge Regression cv模型的均方根误差为(Validation):",np.sqrt(metrics.mean_squared_error(y_val, y_rcv_pred)) )
def save_trials(trials):
pickle.dump(trials, open(f'trials_{len(trials.trials)}.p', 'wb'))<compute_test_metric>
start_time = time.time()
regr_ridge= make_pipeline(RobustScaler() , linear_model.Ridge(alpha = 4.7))
regr_ridge.fit(X_tra,y_tra)
 y_tra_ridge_pred=regr_ridge.predict(X_tra)
 print("The MSE on train set:",np.sqrt(metrics.mean_squared_error(y_tra,y_tra_ridge_pred)))
 y_val_ridge_pred = regr_ridge.predict(X_val)
 print("The MSE on validation set:",np.sqrt(metrics.mean_squared_error(y_val, y_val_ridge_pred)))
 regr_mse = cross_val_score(regr_ridge, X_train, y_train, scoring='neg_mean_squared_error',cv=5)
 print("MSE on Cross-validation:
",np.sqrt(-regr_mse))
print("The mean MSE on Cross-validation is {:.3f}".format(np.mean(np.sqrt(-regr_mse))))
print("--- %s seconds ---" %(time.time() - start_time))
plt.figure('ridge_regr')
 plt.plot([np.expm1(y_val.min()),np.expm1(y_val.max())],[np.expm1(y_val.min()),np.expm1(y_val.max())],'k--',lw=2)
 plt.scatter(np.expm1(y_val),np.expm1(y_val_ridge_pred))
plt.xlabel('Fact Price')
 plt.ylabel('Predict Price')
 plt.title('Predict Effect Using Ridge Regression')
 plt.show()
def save_trials_and_call_objective(hparams, objective, trials, df, kf):
print(f'save this len of trials: {len(trials.trials)}')
 save_trials(trials)
 loss = objective(df, kf, hparams)
 return loss<define_variables>
lscv = LassoCV(alphas=np.arange(0.0001,0.0005,0.00001))
lscv.fit(X_tra,y_tra.values.ravel())
 print('Best parameters found by randomised search:', lscv.alpha_, '
' )
trials_file_name = '.. /input/ntrialsanneal17/anneal_trials_132.p'<load_pretrained>
start_time = time.time()
lasso= make_pipeline(RobustScaler() ,linear_model.Lasso(alpha=0.00043999999999999996))
lasso.fit(X_tra,y_tra)
 y_tra_lasso_pred=lasso.predict(X_tra)
 print("The MSE on train set:",np.sqrt(metrics.mean_squared_error(y_tra,y_tra_lasso_pred)))
 y_val_lasso_pred = lasso.predict(X_val)
 print("The MSE on validation set:",np.sqrt(metrics.mean_squared_error(y_val, y_val_lasso_pred)))
 regr_mse = cross_val_score(lasso, X_train, y_train, scoring='neg_mean_squared_error',cv=5)
 print("MSE on Cross-validation:
",np.sqrt(-regr_mse))
print("The mean MSE on Cross-validation is {:.3f}".format(np.mean(np.sqrt(-regr_mse))))
print("--- %s seconds ---" %(time.time() - start_time))
plt.figure('ridge_regr')
 plt.plot([np.expm1(y_val.min()),np.expm1(y_val.max())],[np.expm1(y_val.min()),np.expm1(y_val.max())],'k--',lw=2)
 plt.scatter(np.expm1(y_val),np.expm1(y_val_lasso_pred))
plt.xlabel('Fact Price')
 plt.ylabel('Predict Price')
 plt.title('Predict Effect Using Lasso Regression')
 plt.show()
try:
trials = load_trials(trials_file_name, True)
 print('Found trials file')
 except FileNotFoundError as e:
trials = ho.Trials()
print('Not found trials file')
 len(trials.trials )<define_variables>
start_time = time.time()
forest = make_pipeline(RobustScaler() , RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,
max_features=0.95, max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=3, min_samples_split=5,
min_weight_fraction_leaf=0.0, n_estimators=100,
n_jobs=None, oob_score=True, random_state=23,
verbose=0, warm_start=False))
forest.fit(X_tra,y_tra.values.ravel())
 y_tra_forest_pred=forest.predict(X_tra)
 print("The MSE on train set:",np.sqrt(metrics.mean_squared_error(y_tra, y_tra_forest_pred)))
 y_val_forest_pred = forest.predict(X_val)
 print("The MSE on validation set:",np.sqrt(metrics.mean_squared_error(y_val, y_val_forest_pred)))
 regr_mse = cross_val_score(forest, X_train, y_train, scoring='neg_mean_squared_error',cv=5)
 print("MSE on Cross-validation:
",np.sqrt(-regr_mse))
print("The mean MSE on Cross-validation is {:.3f}".format(np.mean(np.sqrt(-regr_mse))))
print("--- %s seconds ---" %(time.time() - start_time))
plt.figure('forest')
 plt.plot([np.expm1(y_val.min()),np.expm1(y_val.max())],[np.expm1(y_val.min()),np.expm1(y_val.max())],'k--',lw=2)
 plt.scatter(np.expm1(y_val),np.expm1(y_val_forest_pred))
plt.xlabel('Fact Price')
 plt.ylabel('Predict Price')
 plt.title('Predict Effect Using RandomForest')
 plt.show()
MAX_LENGTH = 200
BATCH_SIZE = 16
EPOCHS = 5<categorify>
start_time = time.time()
xgb = make_pipeline(RobustScaler() ,XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.82, gamma=0,
importance_type='gain', learning_rate=0.2,
max_delta_step=0, max_depth=5, min_child_weight=7, missing=None,
n_estimators=200, n_jobs=-1, nthread=None, objective='reg:squarederror',
random_state=3, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,
seed=None, silent=None, subsample=0.7526315789473684, verbosity=1))
xgb.fit(X_tra,y_tra.values.ravel())
 print("Validation:", xgb.score(X_val,y_val))
y_tra_xgb_pred=xgb.predict(X_tra)
 print("The MSE on train set:",np.sqrt(metrics.mean_squared_error(y_tra, y_tra_xgb_pred)))
 y_val_xgb_pred = xgb.predict(X_val)
 print("The MSE on validation set:",np.sqrt(metrics.mean_squared_error(y_val, y_val_xgb_pred)))
 regr_mse = cross_val_score(xgb, X_train, y_train, scoring='neg_mean_squared_error',cv=5)
 print("MSE on Cross-validation:
",np.sqrt(-regr_mse))
print("The mean MSE on Cross-validation is {:.3f}".format(np.mean(np.sqrt(-regr_mse))))
print("--- %s seconds ---" %(time.time() - start_time))
plt.figure('XGBoost')
 plt.plot([np.expm1(y_val.min()),np.expm1(y_val.max())],[np.expm1(y_val.min()),np.expm1(y_val.max())],'k--',lw=2)
 plt.scatter(np.expm1(y_val),np.expm1(y_val_xgb_pred))
plt.xlabel('Fact Price')
 plt.ylabel('Predict Price')
 plt.title('Predict Effect Using XGBoost')
 plt.show()
def encode_with_tokinizer(data, tokenizer, maximum_length):
input_ids = []
attention_mask = []
for i in range(len(data)) :
encoded = tokenizer.encode_plus(
 data[i],
add_special_tokens=True,
max_length=maximum_length,
pad_to_max_length=True,
return_attention_mask=True,
return_token_type_ids=False 
)
 input_ids.append(encoded['input_ids'])
 attention_mask.append(encoded['attention_mask'])
 return np.array(input_ids), np.array(attention_mask )<define_search_space>
start_time = time.time()
lgb = LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 lgb.fit(X_tra,y_tra.values.ravel())
 print("Validation:", lgb.score(X_val,y_val))
y_tra_lgb_pred=lgb.predict(X_tra)
 print("The MSE on train set:",np.sqrt(metrics.mean_squared_error(y_tra, y_tra_lgb_pred)))
 y_val_lgb_pred = lgb.predict(X_val)
 print("The MSE on validation set:",np.sqrt(metrics.mean_squared_error(y_val, y_val_lgb_pred)))
 regr_mse = cross_val_score(lgb, X_train, y_train, scoring='neg_mean_squared_error',cv=5)
 print("MSE on Cross-validation:
",np.sqrt(-regr_mse))
print("The mean MSE on Cross-validation is {:.3f}".format(np.mean(np.sqrt(-regr_mse))))
print("--- %s seconds ---" %(time.time() - start_time))
plt.figure('lgb')
 plt.plot([np.expm1(y_val.min()),np.expm1(y_val.max())],[np.expm1(y_val.min()),np.expm1(y_val.max())],'k--',lw=2)
 plt.scatter(np.expm1(y_val),np.expm1(y_val_lgb_pred))
plt.xlabel('Fact Price')
 plt.ylabel('Predict Price')
 plt.title('Predict Effect Using LightGBM')
 plt.show()
def node_params(n_layers):
params = {}
params['pack_size'] = n_layers
for n in range(n_layers):
params['n_nodes_layer_{}'.format(n)] = ho.hp.quniform('n_nodes_{}_{}'.format(n_layers, n), 10, 2000, 25)
 params['dropout_layer_{}'.format(n)] = ho.hp.quniform('dropout_{}_{}'.format(n_layers, n), 0, 0.6, 0.05)
 return params<choose_model_class>
start_time = time.time()
svr = SVR(kernel = 'poly')
 svr.fit(X_tra,y_tra.values.ravel())
 print("Validation:", svr.score(X_val,y_val))
y_tra_svr_pred=svr.predict(X_tra)
 print("The MSE on train set:",np.sqrt(metrics.mean_squared_error(y_tra, y_tra_svr_pred)))
 y_val_svr_pred = svr.predict(X_val)
 print("The MSE on validation set:",np.sqrt(metrics.mean_squared_error(y_val, y_val_svr_pred)))
 regr_mse = cross_val_score(svr, X_train, y_train, scoring='neg_mean_squared_error',cv=5)
 print("MSE on Cross-validation:
",np.sqrt(-regr_mse))
print("The mean MSE on Cross-validation is {:.3f}".format(np.mean(np.sqrt(-regr_mse))))
print("--- %s seconds ---" %(time.time() - start_time))
plt.figure('svr')
 plt.plot([np.expm1(y_val.min()),np.expm1(y_val.max())],[np.expm1(y_val.min()),np.expm1(y_val.max())],'k--',lw=2)
 plt.scatter(np.expm1(y_val),np.expm1(y_val_svr_pred))
plt.xlabel('Fact Price')
 plt.ylabel('Predict Price')
 plt.title('Predict Effect Using SVR')
 plt.show()
def create_model(transformer_model, hparams):
input_ids = tf.keras.Input(shape=(MAX_LENGTH,),dtype='int32')
 attention_mask = tf.keras.Input(shape=(MAX_LENGTH,), dtype='int32')
 transformer = transformer_model([input_ids, attention_mask])
 hidden_states = transformer[1]
if hparams['hidden_states_size'] == 1:
output = hidden_states[-1]
else:
hiddes_states_ind = list(range(-hparams['hidden_states_size'], 0, 1))
output = tf.keras.layers.concatenate(tuple([hidden_states[i] for i in hiddes_states_ind]))
if hparams['layers']['pack_size'] >= 1:
output = tf.keras.layers.Dense(hparams['layers']['n_nodes_layer_0'], activation='relu' )(output)
 output = tf.keras.layers.BatchNormalization()(output)
 output = tf.keras.layers.Dropout(hparams['layers']['dropout_layer_0'] )(output)
 if hparams['layers']['pack_size'] >= 2:
output = tf.keras.layers.Dense(hparams['layers']['n_nodes_layer_1'], activation='relu' )(output)
 output = tf.keras.layers.BatchNormalization()(output)
 output = tf.keras.layers.Dropout(hparams['layers']['dropout_layer_1'] )(output)
 if hparams['layers']['pack_size'] >= 3:
output = tf.keras.layers.Dense(hparams['layers']['n_nodes_layer_2'], activation='relu' )(output)
 output = tf.keras.layers.BatchNormalization()(output)
 output = tf.keras.layers.Dropout(hparams['layers']['dropout_layer_2'] )(output)
 output = tf.keras.layers.Dense(1, activation='sigmoid' )(output)
 model = tf.keras.models.Model(inputs = [input_ids, attention_mask], outputs = output)
 model.compile(tf.keras.optimizers.Adam(lr=hparams['lr_rate']), loss='binary_crossentropy', metrics=['accuracy'])
 return model<load_pretrained>
def stacked(all_data):
stacked_pred=0.1*regr.predict(all_data)+0.1*regr_ridge.predict(all_data)+\
0.1*lasso.predict(all_data)+0.1*forest.predict(all_data)+0.2*xgb.predict(all_data)+\
0.2*lgb.predict(all_data)+0.2*svr.predict(all_data)
 return stacked_pred
tokenizers = {
'bert': trfo.BertTokenizer.from_pretrained('bert-large-uncased', do_lower_case=True),
'roberta': trfo.RobertaTokenizer.from_pretrained('roberta-large', do_lower_case=True),
'distilbert': trfo.DistilBertTokenizer.from_pretrained('distilbert-base-uncased', do_lower_case=True)
 }<load_pretrained>
Prediction_final=np.expm1(stacked(X_test))
<choose_model_class><EOS> 
sub = pd.DataFrame()
sub['Id'] = test_data['Id']
sub['SalePrice'] = Prediction_final
print(sub.head())
 sub.to_csv('submission.csv',index=False )