Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_from_url>
%matplotlib inline
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<import_modules>
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
import tokenization
import matplotlib.pyplot as plt
import seaborn as sns
import re
import nltk
import spacy
import sys
import random
import fuzzywuzzy
from fuzzywuzzy import process
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from spacy.util import minibatch
from tensorflow import math
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub<load_from_csv>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv',index_col=0)
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv',index_col=0)
 print(train.shape)
 train.head()
random_seed = 0
data = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 print(data.shape)
 data.head()<count_unique_values>
df_all = pd.concat([train[train.columns[:-1]],test])
 df_all.head(5 )
print(data.isnull().sum())
 display(data.nunique() )<categorify>
y = train.SalePrice
def utils_preprocess_text(text, flg_stemm=False, flg_lemm=True, lst_stopwords=None):
text = re.sub(r"https?://\S+|www\.\S+", "", text)
 html = re.compile(r"<.*?>|&([a-z0-9]+|
text = re.sub(html, "", text)
 l = len(text)
 t = text
text= re.sub(r'[^\x00-\x7f]',r'', text)
 text = re.sub(r'[^\w\s]', '', str(text ).lower().strip())
 lst_text = text.split()
if lst_stopwords is not None:
lst_text = [word for word in lst_text if word not in
lst_stopwords]
if flg_stemm == True:
ps = nltk.stem.porter.PorterStemmer()
lst_text = [ps.stem(word)for word in lst_text]
if flg_lemm == True:
lem = nltk.stem.wordnet.WordNetLemmatizer()
lst_text = [lem.lemmatize(word)for word in lst_text]
text = " ".join(lst_text)
 return text<feature_engineering>
y_original = y.copy()
y = np.log(y )
lst_stopwords = nltk.corpus.stopwords.words("english")
 data["text_clean"] = data["text"].apply(lambda x: utils_preprocess_text(x, flg_stemm=False, flg_lemm=True, lst_stopwords=lst_stopwords))
data.head()<feature_engineering>
df_all.drop(['Alley','FireplaceQu','PoolQC','Fence','MiscFeature'],axis=1,inplace=True )
def clean_location(data):
data['location'] = data['location'].str.lower()
data['location'] = data['location'].str.strip()
data['location'] = data['location'].apply(lambda x: re.sub(r',(?s ).*$', r'', x)if str(x)!= str(np.nan)else np.nan)
 return data<drop_column>
df_all = df_all.replace({"MSSubClass" : {20 : "SC20", 30 : "SC30", 40 : "SC40", 45 : "SC45",
50 : "SC50", 60 : "SC60", 70 : "SC70", 75 : "SC75",
80 : "SC80", 85 : "SC85", 90 : "SC90", 120 : "SC120",
150 : "SC150", 160 : "SC160", 180 : "SC180", 190 : "SC190"}} )
data = clean_location(data )<categorify>
df_all = df_all.replace({"MoSold" : {1 : "January", 2 : "February", 3 : "March", 4 : "April",
5 : "May", 6 : "June", 7 : "July", 8 : "August",
9 : "September", 10 : "October", 11 : "November", 12 : "December"}} )
def replace_matches_in_column(df, column, string_to_match, min_ratio = 90):
strings = df[column].unique()
matches = fuzzywuzzy.process.extract(string_to_match, strings, limit=10, scorer=fuzzywuzzy.fuzz.token_sort_ratio)
 close_matches = [matches[0] for matches in matches if matches[1] >= min_ratio]
rows_with_matches = df[column].isin(close_matches)
 df.loc[rows_with_matches, column] = string_to_match<sort_values>
cat_feats = df_all.dtypes[df_all.dtypes == "object"].index.tolist()
print(str(len(cat_feats)) + ' categoric features' )
locations = data.groupby('location' ).location.count().sort_values(ascending=False)
 locations = locations[locations.values > 3]
locations = locations.index
for loc in locations:
replace_matches_in_column(df=data, column='location', string_to_match=loc )<feature_engineering>
num_feats = df_all.dtypes[df_all.dtypes != "object"].index.tolist()
print(str(len(num_feats)) + ' numeric features' )
def rename_location(data):
data.loc[(data.location == 'united states'), 'location'] = 'usa'
data.loc[(data.location == 'united states of america'), 'location'] = 'usa'
data.loc[(data.location == 'us'), 'location'] = 'usa'
data.loc[(data.location == 'u.s.'), 'location'] = 'usa'
data.loc[(data.location == 'u.s.a'), 'location'] = 'usa'
data.loc[(data.location == 'u.s.a.'), 'location'] = 'usa'
data.loc[(data.location == 'new york'), 'location'] = 'nyc'
data.loc[(data.location == 'ny'), 'location'] = 'nyc'
data.loc[(data.location == 'new york city'), 'location'] = 'nyc'
data.loc[(data.location == 'uk'), 'location'] = 'england'
data.loc[(data.location == 'uk.'), 'location'] = 'england'
data.loc[(data.location == 'united kingdom'), 'location'] = 'england'
data.loc[(data.location == '?? | pittsburgh'), 'location'] = 'pittsburgh'
data.loc[(data.location == '??????'), 'location'] = np.nan
data.loc[(data.location == '?????'), 'location'] = np.nan
data.loc[(data.location == '??'), 'location'] = np.nan
data.loc[(data.location == 'world'), 'location'] = np.nan
data.loc[(data.location == 'world wide'), 'location'] = np.nan
data.loc[(data.location == 'worldwide'), 'location'] = np.nan
data.loc[(data.location == 'everywhere'), 'location'] = np.nan
data.loc[(data.location == 'everywhere.. '), 'location'] = np.nan
data.loc[(data.location == 'earth'), 'location'] = np.nan
data.loc[(data.location == 'happily married with 2 kids'), 'location'] = np.nan
data.loc[(data.location == 'road to the billionaires club'), 'location'] = np.nan
data.loc[(data.location == 'pedophile hunting ground'), 'location'] = np.nan
data.loc[(data.location == 'does it really matter!'), 'location'] = np.nan
data.loc[(data.location == 'planet earth'), 'location'] = np.nan
data.loc[(data.location == 'in the word of god'), 'location'] = np.nan
data.loc[(data.location == 'mad as hell'), 'location'] = np.nan
data.loc[(data.location == 'global'), 'location'] = np.nan
data.loc[(data.location == '
data.loc[(data.location == '304'), 'location'] = np.nan
data.loc[(data.location == ''), 'location'] = np.nan
return data<sort_values>
for i in cat_feats_missing:
print(i)
 print(df_all[i].value_counts(dropna=False))
print("" )
locations = data.groupby('location' ).location.count().sort_values(ascending=False)
 locations = locations[locations.values <= 3]
locations = locations.index
for loc in locations:
data.loc[(data.location == loc), 'location'] = np.nan
data.groupby('location' ).location.count().sort_values(ascending=False )<feature_engineering>
df_all.MSZoning.fillna('RL', inplace = True)
 df_all.drop('Utilities',axis=1,inplace=True)
 df_all.Exterior1st.fillna('VinylSd', inplace = True)
 df_all.Exterior2nd.fillna('VinylSd', inplace = True)
 df_all.MasVnrType.fillna('None', inplace = True)
 df_all.BsmtQual.fillna('No basement',inplace = True)
 df_all.BsmtCond.fillna('No basement',inplace = True)
 df_all.BsmtExposure.fillna('No basement',inplace = True)
 df_all.BsmtFinType1.fillna('No basement',inplace = True)
 df_all.BsmtFinType2.fillna('No basement',inplace = True)
 df_all.Electrical.fillna('SBrkr',inplace = True)
 df_all.KitchenQual.fillna('TA',inplace = True)
 df_all.Functional.fillna('Typ',inplace = True)
 df_all.GarageType.fillna('No garage',inplace = True)
 df_all.GarageFinish.fillna('No garage',inplace = True)
 df_all.GarageQual.fillna('No garage',inplace = True)
 df_all.GarageCond.fillna('No garage',inplace = True)
 df_all.SaleType.fillna('WD',inplace = True)
 df_all.head(5 )
def concatenate(data):
data['sequence'] = data['text_clean'].map(str)+ ' XXLOC ' + data['location'].map(str)\
+ ' XXKEY ' + data['keyword'].map(str)+ ' TWTXX'
return data
data = concatenate(data)
 print(data.sequence[0])
 print(data.sequence[98])
 print(data.sequence[100] )<feature_engineering>
df_all = df_all.replace({"Alley" : {"Grvl" : 1, "Pave" : 2},
"BsmtCond" : {"No basement" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"BsmtExposure" : {"No basement" : 0, "Mn" : 1, "Av": 2, "Gd" : 3},
"BsmtFinType1" : {"No basement" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3, "BLQ" : 4,
"ALQ" : 5, "GLQ" : 6},
"BsmtFinType2" : {"No basement" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3, "BLQ" : 4,
"ALQ" : 5, "GLQ" : 6},
"BsmtQual" : {"No basement" : 0, "Po" : 1, "Fa" : 2, "TA": 3, "Gd" : 4, "Ex" : 5},
"ExterCond" : {"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"ExterQual" : {"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"FireplaceQu" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"Functional" : {"Sal" : 1, "Sev" : 2, "Maj2" : 3, "Maj1" : 4, "Mod": 5,
"Min2" : 6, "Min1" : 7, "Typ" : 8},
"GarageCond" : {"No garage" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"GarageQual" : {"No garage" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"HeatingQC" : {"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"KitchenQual" : {"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"LandSlope" : {"Sev" : 1, "Mod" : 2, "Gtl" : 3},
"LotShape" : {"IR3" : 1, "IR2" : 2, "IR1" : 3, "Reg" : 4},
"PavedDrive" : {"N" : 0, "P" : 1, "Y" : 2},
"PoolQC" : {"No" : 0, "Fa" : 1, "TA" : 2, "Gd" : 3, "Ex" : 4},
"Street" : {"Grvl" : 1, "Pave" : 2},
"Utilities" : {"ELO" : 1, "NoSeWa" : 2, "NoSewr" : 3, "AllPub" : 4}}
)
test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 test["text_clean"] = test["text"].apply(lambda x: utils_preprocess_text(x, flg_stemm=False, flg_lemm=True, lst_stopwords=lst_stopwords))
test = clean_location(test)
 locations = test.groupby('location' ).location.count().sort_values(ascending=False)
 locations = locations.index
for loc in locations:
replace_matches_in_column(df=test, column='location', string_to_match=loc)
 locations = data.groupby('location' ).location.count().sort_values(ascending=False)
 locations = locations.index
for loc in locations:
replace_matches_in_column(df=test, column='location', string_to_match=loc)
 test = rename_location(test)
 test = concatenate(test )<categorify>
df_all.drop('BsmtCond',axis=1,inplace=True)
 df_all.drop('GarageCond',axis=1,inplace=True )
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<choose_model_class>
cat_feats = df_all.dtypes[df_all.dtypes == "object"].index.tolist()
print(str(len(cat_feats)) + ' categoric features')
 print("")
 for i in cat_feats:
print(i)
 print(df_all[i].value_counts(dropna=False))
print("" )
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<choose_model_class>
cat_feats = df_all.dtypes[df_all.dtypes == "object"].index.tolist()
print(str(len(cat_feats)) + ' categoric features')
 df_all = pd.get_dummies(df_all)
 df_all.head(5 )
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )<feature_engineering>
df_all[null_columns].isnull().sum()
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
to_set_to_zero = ['MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','BsmtFullBath','BsmtHalfBath','GarageCars','GarageArea']
df_all[to_set_to_zero] = df_all[to_set_to_zero].fillna(0)
 df_all.drop('GarageYrBlt',axis=1,inplace = True)
 df_all = df_all.fillna(df_all.median() )
train_input = bert_encode(data.sequence.values, tokenizer, max_len=65)
 test_input = bert_encode(test.sequence.values, tokenizer, max_len=65)
 train_labels = data.target.values<train_model>
num_feats.remove('GarageYrBlt' )
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.15,
epochs=3,
callbacks=[checkpoint],
batch_size=8
)<predict_on_test>
skewed_feats = df_all[num_feats].apply(lambda x: skew(x))
skewed_feats.sort_values()
model.load_weights('model.h5')
 test_pred = model.predict(test_input )<save_to_csv>
skewed_feats = skewed_feats[skewed_feats > 0.75].index
skewed_feats
submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False )<load_from_csv>
skewed_feats=skewed_feats.tolist()
skewed_feats = [e for e in skewed_feats if e not in ['GrLivArea','LotArea','1stFlrSF']]
path_data_test='/kaggle/input/nlp-getting-started/test.csv'
test_data=pd.read_csv(path_data_test)
 path_data_train='/kaggle/input/nlp-getting-started/train.csv'
train_data=pd.read_csv(path_data_train )<categorify>
pt = PowerTransformer('yeo-johnson',standardize=False)
 print(pt.fit(df_all[skewed_feats]))
print("")
 print(pt.lambdas_ )
def clean(text):
text = re.sub(r"
","",text)
 text = text.lower()
text = re.sub(r"\d","",text)
 text = re.sub(r'[^\x00-\x7f]',r' ',text)
 text = re.sub(r'[^\w\s]','',text)
 text = re.sub(r'http\S+|www.\S+', '', text)
 return text 
<feature_engineering>
df_all[skewed_feats] = pt.transform(df_all[skewed_feats])
 df_all[['GrLivArea','LotArea','1stFlrSF']]=df_all[['GrLivArea','LotArea','1stFlrSF']].apply(np.log)
 skewed_feats = skewed_feats + ['GrLivArea','LotArea','1stFlrSF']
df_all[skewed_feats].skew().sort_values()
train_data['cleaned'] = train_data['text'].apply(lambda x : clean(x))
test_data['cleaned']= test_data['text'].apply(lambda x : clean(x)) 
<prepare_x_and_y>
pd.concat([X_train,y],axis=1 ).corr().iloc[:,-1].sort_values(ascending=False ).head(10 )
tweets_pipeline = Pipeline([('CVec', CountVectorizer(stop_words='english')) , 
('Tfidf', TfidfTransformer())])
 X=train_data['cleaned'].to_numpy()
Y=train_data['target'].to_numpy()
X_train_tranformed = tweets_pipeline.fit_transform(X)
 X_test=test_data['cleaned']
X_test_tranformed = tweets_pipeline.transform(X_test)
 train_x,val_x,train_y,val_y=train_test_split(X,Y,random_state=0)
 model=KNeighborsClassifier(n_neighbors=7)
 model.fit(X_train_tranformed,Y)
 preds=model.predict(X_test_tranformed)
 output = pd.DataFrame({'id': test_data.id, 'target': preds})
 output.to_csv('sample_submission.csv', index=False)
<load_from_csv>
index_to_drop = X_train[(X_train['GrLivArea']>8.3)&(y<12.5)].index.tolist()
X_train = X_train.drop(index_to_drop,axis=0)
 y = y.drop(index_to_drop)
 index_to_drop = X_train[(X_train['GrLivArea']>6.5)&(y<10.7)].index.tolist()
X_train = X_train.drop(index_to_drop,axis=0)
 y = y.drop(index_to_drop )
data=pd.read_csv('.. /input/train.csv')
 out_data=pd.read_csv('.. /input/test.csv')
<prepare_x_and_y>
scaler = RobustScaler()
scaler.fit(X_train)
 scaled_features_train = scaler.transform(X_train)
 scaled_features_test = scaler.transform(X_test)
 X_train = pd.DataFrame(scaled_features_train,columns = X_train.columns)
 X_test = pd.DataFrame(scaled_features_test,index = X_test.index, columns = X_test.columns)
 X_train.head(5 )
x=data.loc[:,'var3':'var38']
y=data['TARGET']<prepare_x_and_y>
from sklearn.metrics import mean_squared_error, make_scorer
from sklearn.model_selection import cross_val_score, train_test_split
from sklearn.linear_model import LinearRegression, Ridge, RidgeCV, Lasso, LassoCV, ElasticNetCV
x=np.asarray(x)
 y=np.asarray(y )<import_modules>
Xt_train, Xt_test, yt_train, yt_test, = train_test_split(X_train, y, test_size = 0.3 )
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import metrics
from sklearn.cross_validation import train_test_split<load_from_csv>
def evaluate_model(model):
yt_train_pred = model.predict(Xt_train)
 yt_test_pred = model.predict(Xt_test)
 rmse_train = np.sqrt(mean_squared_error(yt_train,yt_train_pred))
rmse_test = np.sqrt(mean_squared_error(yt_test,yt_test_pred))
print("RMSE on Training set :", rmse_train)
 print("RMSE on Test set :", rmse_test)
 plt.figure(figsize=[20,8])
 plt.subplot(1,2,1)
 plt.scatter(yt_train_pred, yt_train_pred - yt_train, c = "blue", marker = "s", label = "Training data")
 plt.scatter(yt_test_pred, yt_test_pred - yt_test, c = "lightgreen", marker = "s", label = "Validation data")
 plt.title(model)
 plt.xlabel("Predicted values")
 plt.ylabel("Residuals")
 plt.legend(loc = "upper left")
 plt.hlines(y = 0, xmin = 10.5, xmax = 13.5, color = "red")
 plt.subplot(1,2,2)
 plt.scatter(yt_train_pred, yt_train, c = "blue", marker = "s", label = "Training data")
 plt.scatter(yt_test_pred, yt_test, c = "lightgreen", marker = "s", label = "Validation data")
 plt.title(model)
 plt.xlabel("Predicted values")
 plt.ylabel("Real values")
 plt.legend(loc = "upper left")
 plt.plot([10.5, 13.5], [10.5, 13.5], c = "red" )
matplotlib.use("Agg")
 training = pd.read_csv(".. /input/train.csv", index_col=0)
 test = pd.read_csv(".. /input/test.csv", index_col=0)
 print(training.shape)
 print(test.shape)
 training = training.replace(-999999,2)
 X = training.iloc[:,:-1]
y = training.TARGET
X['n0'] =(X == 0 ).sum(axis=1)
 p = 86
p = 80
p = 77
p = 75
X_bin = Binarizer().fit_transform(scale(X))
selectChi2 = SelectPercentile(chi2, percentile=p ).fit(X_bin, y)
 selectF_classif = SelectPercentile(f_classif, percentile=p ).fit(X, y)
 chi2_selected = selectChi2.get_support()
chi2_selected_features = [ f for i,f in enumerate(X.columns)if chi2_selected[i]]
print('Chi2 selected {} features {}.'.format(chi2_selected.sum() ,
chi2_selected_features))
f_classif_selected = selectF_classif.get_support()
f_classif_selected_features = [ f for i,f in enumerate(X.columns)if f_classif_selected[i]]
print('F_classif selected {} features {}.'.format(f_classif_selected.sum() ,
f_classif_selected_features))
selected = chi2_selected & f_classif_selected
print('Chi2 & F_classif selected {} features'.format(selected.sum()))
features = [ f for f,s in zip(X.columns, selected)if s]
print(features )<train_model>
def cross_validation_evaluation(model):
scores = -cross_val_score(model, Xt_train, yt_train, cv=10,scoring="neg_mean_squared_error")
 return scores.mean()
X_sel = X[features]
X_train, X_test, y_train, y_test = \
cross_validation.train_test_split(X_sel, y, random_state=1301, stratify=y, test_size=0.3)
 clf = xgb.XGBClassifier(missing=9999999999,
max_depth = 8,
n_estimators=1000,
learning_rate=0.02,
nthread=4,
subsample=0.8,
colsample_bytree=0.5,
min_child_weight = 7,
seed=4242)
 clf.fit(X_train, y_train, early_stopping_rounds=50, eval_metric="auc",
eval_set=[(X_train, y_train),(X_test, y_test)])
 print('Overall AUC:', roc_auc_score(y, clf.predict_proba(X_sel, ntree_limit=clf.best_iteration)[:,1]))
test['n0'] =(test == 0 ).sum(axis=1)
 sel_test = test[features]
y_pred = clf.predict_proba(sel_test, ntree_limit=clf.best_iteration)
 submission = pd.DataFrame({"ID":test.index, "TARGET":y_pred[:,1]})
 submission.to_csv("submission.csv", index=False)
 mapFeat = dict(zip(["f"+str(i)for i in range(len(features)) ],features))
ts = pd.Series(clf.booster().get_fscore())
 ts.sort_values() [-15:].plot(kind="barh", title=("features importance"))
featp = ts.sort_values() [-15:].plot(kind='barh', x='feature', y='fscore', legend=False, figsize=(6, 10))
plt.title('XGBoost Feature Importance')
 fig_featp = featp.get_figure()
fig_featp.savefig('feature_importance_xgb.png', bbox_inches='tight', pad_inches=1 )<compute_train_metric>
linear_regression_model = LinearRegression()
linear_regression_model.fit(Xt_train, yt_train )
a=clf.booster().get_fscore()<sort_values>
evaluate_model(linear_regression_model )
sorted_x = sorted(a.items() , key=operator.itemgetter(1))<prepare_x_and_y>
ridge_regression_model = Ridge(alpha=1.0)
 ridge_regression_model.fit(Xt_train, yt_train )
fe=['num_var22_hace3','num_med_var45_ult3','saldo_var5','saldo_var42','num_var45_hace3','num_var22_ult3','saldo_medio_var5_ult1','num_var45_ult3','saldo_medio_var5_hace2',
'saldo_var30',
'saldo_medio_var5_ult3',
'saldo_medio_var5_hace3',
'var15',
'var38']
x=data[fe]<train_model>
evaluate_model(ridge_regression_model )
gb=GradientBoostingClassifier()
fe=np.asarray(fe)
 gb.fit(fe,y )<train_model>
ridge_regression_improved_model = Ridge(alpha=optimised_alpha)
 ridge_regression_improved_model.fit(Xt_train, yt_train )
X_fit, X_eval, y_fit, y_eval= cv.train_test_split(X, y, test_size=0.3)
 clf = xgb.XGBClassifier(missing=np.nan, max_depth=5, n_estimators=570, learning_rate=0.02, nthread=4, subsample=0.68, colsample_bytree=0.7, seed=1235)
 clf.fit(X, y, early_stopping_rounds=500, eval_metric="auc", eval_set=[(X_eval, y_eval)])
<save_to_csv>
evaluate_model(ridge_regression_improved_model )
print('Overall AUC:', metrics.roc_auc_score(y, clf.predict_proba(X)[:,1]))
X_test = test_dataset[feature_names]
y_pred= clf.predict_proba(X_test)[:,1]
submission = pd.DataFrame({"ID":X_test.index, "TARGET":y_pred})
 submission.to_csv("submission.csv", index=False)
<feature_engineering>
lasso_regression_model = Lasso()
lasso_regression_model.fit(Xt_train, yt_train )
%matplotlib inline
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 sns.countplot(x="TARGET", data=train_df)
 for feat in train_df.columns:
if train_df[feat].dtype == 'float64':
train_df[feat][np.isnan(train_df[feat])] = train_df[feat].mean()
test_df[feat][np.isnan(test_df[feat])] = test_df[feat].mean()
elif train_df[feat].dtype == 'object':
train_df[feat][train_df[feat] != train_df[feat]] = train_df[feat].value_counts().index[0]
test_df[feat][test_df[feat] != test_df[feat]] = test_df[feat].value_counts().index[0]
for feat in train_df.columns:
if train_df[feat].dtype == 'object':
lbl = preprocessing.LabelEncoder()
lbl.fit(np.unique(list(train_df[feat].values)+ list(test_df[feat].values)))
 train_df[feat] = lbl.transform(list(train_df[feat].values))
test_df[feat] = lbl.transform(list(test_df[feat].values))
X_train = train_df.drop(["ID","TARGET"],axis=1)
 Y_train = train_df["TARGET"]
X_test = test_df.drop("ID",axis=1 ).copy()<prepare_x_and_y>
evaluate_model(lasso_regression_model )
xgtrain = xgb.DMatrix(X_train, Y_train)
 xgtest = xgb.DMatrix(X_test )<split>
lasso_regression_improved_model = Lasso(alpha=optimised_alpha)
 lasso_regression_improved_model.fit(Xt_train, yt_train )
x_train, x_eval, y_train, y_eval = train_test_split(X_train, Y_train, test_size=0.33, stratify=Y_train, random_state=1)
 dtrain = xgb.DMatrix(x_train, y_train)
 deval = xgb.DMatrix(x_eval, y_eval)
 watchlist = [(dtrain, 'train'),(deval, 'eval')]<train_on_grid>
import xgboost as xgb
from xgboost.sklearn import XGBRegressor
from sklearn.model_selection import GridSearchCV
xgboost_params = {
'objective': 'multi:softprob',
'booster': 'gbtree',
'eval_metric': 'auc',
'eta': 0.3,
'subsample': 0.75,
'colsample_bytree': 0.75,
}
num_rounds = 1000
cv = xgb.cv(xgboost_params, xgtrain, metrics='auc', num_boost_round=num_rounds, nfold=5, stratified=True, seed=1, early_stopping_rounds=25, verbose_eval=True,show_stdv=False )<train_model>
xg_reg = xgb.XGBRegressor()
bst = xgb.train(
 xgboost_params,
dtrain,
num_boost_round=num_rounds,
evals=watchlist,
early_stopping_rounds=25,
verbose_eval=True )<save_to_csv>
xg_reg.fit(Xt_train, yt_train )
preds = bst.predict(xgtest, ntree_limit=bst.best_iteration)
 submission = pd.DataFrame()
submission["ID"] = test_df["ID"]
submission["TARGET"] = preds[:,1]
submission.to_csv('santander.csv', index=False )<not_enough_vertices>
evaluate_model(xg_reg )
try:
%tensorflow_version 2.x
except Exception:
pass
lemmatizer = WordNetLemmatizer() 
<set_options>
tf.random.set_seed(123)
 random.seed(123 )<load_pretrained>
p_dict = {'colsample_bytree': 0.8, 'gamma': 0, 'learning_rate': 0.03, 'max_depth': 3, 'min_child_weight': 2, 'n_estimators': 1000, 'objective': 'reg:linear', 'silent': 1, 'subsample': 0.7}
for zip_path in ['.. /input/sentiment-analysis-on-movie-reviews/train.tsv.zip',
'.. /input/sentiment-analysis-on-movie-reviews/test.tsv.zip']:
with ZipFile(zip_path, 'r')as zip:
zip.printdir()
print('Extracting all the files now...')
 zip.extractall()
print('Done!' )<load_from_csv>
xg_reg_improved = xgb.XGBRegressor(objective = p_dict['objective'], colsample_bytree = p_dict['colsample_bytree'],
learning_rate = p_dict['learning_rate'], max_depth = p_dict['max_depth'], min_child_weight = p_dict['min_child_weight'],
n_estimators = p_dict['n_estimators'], silent=p_dict['silent'], subsample=p_dict['subsample'] )
sample = pd.read_csv(".. /input/sentiment-analysis-on-movie-reviews/sampleSubmission.csv")
 train = pd.read_csv("train.tsv", delimiter='\t')
 test = pd.read_csv("test.tsv", delimiter='\t' )<string_transform>
xg_reg_improved.fit(Xt_train, yt_train )
def clean_sentences(df):
reviews = []
for sent in tqdm(df['Phrase']):
review_text = BeautifulSoup(sent ).get_text()
review_text = re.sub("[^a-zA-Z]"," ", review_text)
 words = word_tokenize(review_text.lower())
 lemma_words = [lemmatizer.lemmatize(i)for i in words]
reviews.append(lemma_words)
 return(reviews)
 train_sentences = clean_sentences(train)
 test_sentences = clean_sentences(test)
 print(len(train_sentences))
print(len(test_sentences))<categorify>
evaluate_model(xg_reg_improved )
target=train.Sentiment.values
y_target=to_categorical(target)
 num_classes=y_target.shape[1]<split>
def return_predictions_model(model,inputs):
predictions = model.predict(inputs)
 return predictions
X_train,X_val,y_train,y_val = train_test_split(train_sentences,
y_target,
test_size=0.2,
stratify=y_target )<count_unique_values>
lasso_regression_predict = return_predictions_model(lasso_regression_improved_model,Xt_test)
 xg_reg_predict = return_predictions_model(xg_reg_improved,Xt_test)
 ridge_regression_predict = return_predictions_model(ridge_regression_improved_model,Xt_test )
unique_words = set()
len_max = 0
for sent in tqdm(X_train):
unique_words.update(sent)
 if(len_max<len(sent)) :
len_max = len(sent)
 print(len(list(unique_words)))
 print(len_max )<string_transform>
stacked_predictions = np.array([lasso_regression_predict,xg_reg_predict,ridge_regression_predict])
 stacked_predictions_avg = np.average(stacked_predictions, axis=0)
 rmse_test = np.sqrt(mean_squared_error(yt_test,stacked_predictions_avg))
print("RMSE on Test set(non-weighted):", rmse_test)
 stacked_predictions_weighted =(3*lasso_regression_predict + 2*xg_reg_predict + ridge_regression_predict)/6
rmse_test = np.sqrt(mean_squared_error(yt_test,stacked_predictions_weighted))
print("RMSE on Test set(weighted):", rmse_test)
tokenizer = Tokenizer(num_words=len(list(unique_words)))
 tokenizer.fit_on_texts(list(X_train))
X_train = tokenizer.texts_to_sequences(X_train)
 X_val = tokenizer.texts_to_sequences(X_val)
 X_test = tokenizer.texts_to_sequences(test_sentences)
 X_train = sequence.pad_sequences(X_train, maxlen=len_max)
 X_val = sequence.pad_sequences(X_val, maxlen=len_max)
 X_test = sequence.pad_sequences(X_test, maxlen=len_max)
 print(X_train.shape,X_val.shape,X_test.shape )<choose_model_class>
lasso_regression_improved_model.fit(X_train, y)
 xg_reg_improved.fit(X_train, y)
 ridge_regression_improved_model.fit(X_train, y )
early_stopping = EarlyStopping(min_delta = 0.001, mode = 'max', monitor='val_acc', patience = 2)
 callback = [early_stopping]<choose_model_class>
lasso_regression_predict = return_predictions_model(lasso_regression_improved_model,X_test)
 xg_reg_predict = return_predictions_model(xg_reg_improved,X_test)
 ridge_regression_predict = return_predictions_model(ridge_regression_improved_model,X_test )
model=Sequential()
model.add(Embedding(len(list(unique_words)) ,300,input_length=len_max))
model.add(LSTM(128,dropout=0.5, recurrent_dropout=0.5,return_sequences=True))
model.add(LSTM(64,dropout=0.5, recurrent_dropout=0.5,return_sequences=False))
model.add(Dense(100,activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes,activation='softmax'))
model.compile(loss='categorical_crossentropy',optimizer=Adam(lr=0.005),metrics=['accuracy'])
 model.summary()<train_model>
stacked_predictions_weighted =(3*lasso_regression_predict + 2*xg_reg_predict + ridge_regression_predict)/6
history=model.fit(X_train, y_train, validation_data=(X_val, y_val),epochs=6, batch_size=256, verbose=1, callbacks=callback )<predict_on_test>
d = {'Id': test.index, 'SalePrice': stacked_predictions_weighted}
predictions_df = pd.DataFrame(data=d)
 predictions_df.head(5 )
y_pred=model.predict_classes(X_test )<save_to_csv>
predictions_df.SalePrice = predictions_df.SalePrice.apply(np.exp)
 predictions_df.head(5 )
<load_from_csv><EOS> 
predictions_df.to_csv('output.csv', index=False)
 predictions_df.head(5 )