Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_from_csv>
%matplotlib inline
warnings.filterwarnings("ignore" )
df = pd.read_csv('.. /input/train.csv' )<prepare_x_and_y>
df_train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 data = pd.concat(( df_train, df_test)).reset_index(drop=True)
 data.drop(['SalePrice'], axis=1, inplace=True)
 df_train.head()
X, y = df['question_text'], df['target'].values<string_transform>
correlations = corrmat["SalePrice"].sort_values(ascending=False)
 features = correlations.index[0:10]
features
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(X)
 X_seq = tokenizer.texts_to_sequences(X)
 X_seq = pad_sequences(X_seq, maxlen=maxlen )<load_pretrained>
df_train.drop(['Id'], axis=1, inplace=True)
 df_test.drop(['Id'], axis=1, inplace=True)
 data.drop(['Id'], axis=1, inplace=True )
emb_mat_1, emb_idx_1 = load_glove(tokenizer.word_index, max_features )<load_pretrained>
training_null = pd.isnull(df_train ).sum()
testing_null = pd.isnull(df_test ).sum()
null = pd.concat([training_null, testing_null], axis=1, keys=["Training", "Testing"] )
emb_mat_2, emb_idx_2 = load_para(tokenizer.word_index, max_features )<compute_test_metric>
null_with_meaning = ["Alley", "BsmtQual", "BsmtCond", "BsmtExposure", "BsmtFinType1", "BsmtFinType2", "FireplaceQu", "GarageType", "GarageFinish", "GarageQual", "GarageCond", "PoolQC", "Fence", "MiscFeature"]
emb_mat = np.mean([emb_mat_1, emb_mat_2], axis = 0 )<prepare_x_and_y>
for i in null_with_meaning:
df_train[i].fillna("None", inplace=True)
 df_test[i].fillna("None", inplace=True)
 data[i].fillna("None", inplace=True )
X_train, y_train = X_seq, y<load_from_csv>
null_many = null[null.sum(axis=1)> 200]
null_few = null[(null.sum(axis=1)> 0)&(null.sum(axis=1)< 200)]
null_many
test_df = pd.read_csv('.. /input/test.csv')
 X_real = test_df['question_text'] 
<string_transform>
df_train.drop("LotFrontage", axis=1, inplace=True)
 df_test.drop("LotFrontage", axis=1, inplace=True)
 data.drop("LotFrontage", axis=1, inplace=True )
X_real = tokenizer.texts_to_sequences(X_real)
 X_real = pad_sequences(X_real, maxlen=maxlen )<choose_model_class>
df_train["GarageYrBlt"].fillna(df_train["GarageYrBlt"].mean() , inplace=True)
 df_test["GarageYrBlt"].fillna(df_test["GarageYrBlt"].mean() , inplace=True)
 data["GarageYrBlt"].fillna(data["GarageYrBlt"].mean() , inplace=True)
 df_train["MasVnrArea"].fillna(df_train["MasVnrArea"].mean() , inplace=True)
 df_test["MasVnrArea"].fillna(df_test["MasVnrArea"].mean() , inplace=True)
 data["MasVnrArea"].fillna(data["MasVnrArea"].mean() , inplace=True)
 df_train["MasVnrType"].fillna("None", inplace=True)
 df_test["MasVnrType"].fillna("None", inplace=True)
 data["MasVnrType"].fillna("None", inplace=True )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<define_search_model>
types_train = df_train.dtypes
num_train = types_train[(types_train == int)|(types_train == float)]
cat_train = types_train[types_train == object]
types_test = df_test.dtypes
num_test = types_test[(types_test == int)|(types_test == float)]
cat_test = types_test[types_test == object]
def model_lstm_atten_bi(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(40, return_sequences=True))(x)
 y = Bidirectional(CuDNNLSTM(40, return_sequences=True))(x)
 atten_1 = Attention(maxlen )(x)
 atten_2 = Attention(maxlen )(y)
 avg_pool = GlobalAveragePooling1D()(y)
 max_pool = GlobalMaxPooling1D()(y)
 conc = concatenate([atten_1, atten_2, avg_pool, max_pool])
 conc = Dense(32, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=[f1])
 return model<define_search_model>
numerical_values_train = list(num_train.index)
 numerical_values_test = list(num_test.index)
 fill_num = numerical_values_train+numerical_values_test
print(fill_num )
def model_lstm_cnn(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix],trainable = False )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(50, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(50, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(32, return_sequences=True))(x)
 x = Conv1D(32, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, max_pool])
 x = Dense(32, activation='relu' )(x)
 x = Dropout(0.1 )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',optimizer='rmsprop',metrics=[f1])
 return model<prepare_x_and_y>
for i in fill_num:
df_train[i].fillna(df_train[i].mean() , inplace=True )
clr = CyclicLR(base_lr=0.001, max_lr=0.004,
step_size=300., mode='exp_range',
gamma=0.99994)
 train_meta = np.zeros(y_train.shape)
 test_meta = np.zeros(X_real.shape[0])
 splits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=14 ).split(X_train, y_train))
for idx,(train_idx, valid_idx)in enumerate(splits):
X_train1 = X_train[train_idx]
y_train1 = y_train[train_idx]
X_val = X_train[valid_idx]
y_val = y_train[valid_idx]
model = model_lstm_atten_bi(emb_mat)
 pred_val_y, pred_test_y, best_score = train_pred(model, X_train1, y_train1, X_val, y_val, epochs = 7, callback = [clr,])
 train_meta[valid_idx] = pred_val_y.reshape(-1)
 test_meta += pred_test_y.reshape(-1)/ len(splits )<save_to_csv>
fill_num.remove('SalePrice' )
search_result = threshold_search(y_train, train_meta)
 print(search_result)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_meta > search_result['threshold']
sub.to_csv("submission.csv", index=False )<import_modules>
for i in fill_num:
df_test[i].fillna(df_test[i].mean() , inplace=True)
 data[i].fillna(data[i].mean() , inplace=True )
import re
import os
import sys
import math
import string
import random
import datetime
import numpy as np
import pandas as pd
from time import time
from tqdm import tqdm, trange
from sklearn import metrics
import keras.layers
from keras import backend as K
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.models import Sequential, Model
from keras.layers import CuDNNLSTM, CuDNNGRU, Dense, Bidirectional, Dropout, BatchNormalization, Activation, Input
from keras.optimizers import Adam
from keras.regularizers import l1_l2
from keras.engine.topology import Layer
from keras.preprocessing.sequence import pad_sequences<train_model>
categorical_values_train = list(cat_train.index)
 categorical_values_test = list(cat_test.index )
print(f'Starting at {datetime.datetime.now() }' )<load_from_csv>
fill_cat = []
for i in categorical_values_train:
if i in list(null_few.index):
fill_cat.append(i)
 print(fill_cat )
df=pd.read_csv('.. /input/train.csv')
 df=df[['question_text','target']]
df=df.dropna()
max_word_len=60<string_transform>
def most_common_term(lst):
lst = list(lst)
 return max(set(lst), key=lst.count)
 most_common = []
for i in fill_cat:
most_common.append(most_common_term(data[i]))
most_common
embedding_path='glove.840B.300d/glove.840B.300d.txt'
vocab={}
with open('.. /input/embeddings/' + embedding_path,'r')as f:
for line_number,line in enumerate(tqdm(f)) :
key,values=line.split(' ')[0],line.split(' ')[1:]
if not any(char in string.punctuation for char in key):
vocab[key]=np.asarray(values,dtype='float32')
 print(f'Total of {len(vocab.keys())} words in vocabulary' )<define_variables>
most_common_dictionary = {fill_cat[0]: [most_common[0]], fill_cat[1]: [most_common[1]], fill_cat[2]: [most_common[2]], fill_cat[3]: [most_common[3]],
fill_cat[4]: [most_common[4]], fill_cat[5]: [most_common[5]], fill_cat[6]: [most_common[6]], fill_cat[7]: [most_common[7]],
fill_cat[8]: [most_common[8]]}
most_common_dictionary
trans_table={key:' ' for key in string.punctuation}
unknown_word=np.zeros(300 )<categorify>
k = 0
for i in fill_cat:
df_train[i].fillna(most_common[k], inplace=True)
 df_test[i].fillna(most_common[k], inplace=True)
 data[i].fillna(most_common[k], inplace=True)
 k += 1
def str_prep(s):
s=s.translate(str.maketrans(trans_table)).lower().strip()
s=re.sub(' +',' ',s)
 s=s.split(' ')
 ar=np.asarray([vocab.get(x,unknown_word)for x in s],dtype='float32')
 return ar<categorify>
training_null = pd.isnull(df_train ).sum()
testing_null = pd.isnull(df_test ).sum()
null = pd.concat([training_null, testing_null], axis=1, keys=["Training", "Testing"])
 null[null.sum(axis=1)> 0]
def batch_gen(df,min_batch_size=1024,transformation_func=str_prep, training_mode=True):
n_batches=math.ceil(len(df)/min_batch_size)
 while True:
df=df.sample(frac=1 ).reset_index(drop=True)
 for batch in range(n_batches):
start=batch*min_batch_size
end=start+min_batch_size
if batch==n_batches-1:
X=df['question_text'][start:].apply(transformation_func)
 X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')
 y=np.array(df['target'][start:])
 else:
X=df['question_text'][start:end].apply(transformation_func)
 X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')
 y=np.array(df['target'][start:end])
 yield X, y<split>
df_train["LogPrice"] = np.log(df_train["SalePrice"])
 df_train.head()
def train_val_split(df,frac=0.2):
df=df.sample(frac=1 ).reset_index(drop=True)
 df_sincere=df[df['target']==0]
df_taxic=df[df['target']==1]
sincere_border=int(len(df_sincere)*(1-frac))
taxic_border=int(len(df_taxic)*(1-frac))
df_train=pd.concat([df_sincere[:sincere_border],df_taxic[:taxic_border]])
 df_val=pd.concat([df_sincere[sincere_border:],df_taxic[taxic_border:]])
 print('Training data peppered(prepared)!')
 return df_train.sample(frac=1 ).reset_index(drop=True), df_val.sample(frac=1 ).reset_index(drop=True )<choose_model_class>
df_train_add = df_train.copy()
df_train_add['TotalSF']=df_train_add['TotalBsmtSF'] + df_train_add['1stFlrSF'] + df_train_add['2ndFlrSF']
df_train_add['Total_Bathrooms'] =(df_train_add['FullBath'] +(0.5 * df_train_add['HalfBath'])+
df_train_add['BsmtFullBath'] +(0.5 * df_train_add['BsmtHalfBath']))
df_train_add['Total_porch_sf'] =(df_train_add['OpenPorchSF'] + df_train_add['3SsnPorch'] +
df_train_add['EnclosedPorch'] + df_train_add['ScreenPorch'] +
df_train_add['WoodDeckSF'])
 df_test_add = df_test.copy()
df_test_add['TotalSF']=df_test_add['TotalBsmtSF'] + df_test_add['1stFlrSF'] + df_test_add['2ndFlrSF']
df_test_add['Total_Bathrooms'] =(df_test_add['FullBath'] +(0.5 * df_test_add['HalfBath'])+
df_test_add['BsmtFullBath'] +(0.5 * df_test_add['BsmtHalfBath']))
df_test_add['Total_porch_sf'] =(df_test_add['OpenPorchSF'] + df_test_add['3SsnPorch'] +
df_test_add['EnclosedPorch'] + df_test_add['ScreenPorch'] +
df_test_add['WoodDeckSF'] )
class Attention(Layer):
def __init__(self, step_dim, W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None, bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],), initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],), initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
df_train_add['haspool'] = df_train_add['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df_train_add['has2ndfloor'] = df_train_add['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df_train_add['hasgarage'] = df_train_add['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 df_train_add['hasbsmt'] = df_train_add['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df_train_add['hasfireplace'] = df_train_add['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 df_test_add['haspool'] = df_test_add['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df_test_add['has2ndfloor'] = df_test_add['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df_test_add['hasgarage'] = df_test_add['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 df_test_add['hasbsmt'] = df_test_add['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df_test_add['hasfireplace'] = df_test_add['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )
<predict_on_test>
df_train[df_train["SalePrice"] > 600000 ]
def get_predictions(df, model, min_batch_size=1024,transformation_func=str_prep):
predictions=np.ndarray(shape=(0,1))
print(f'Total values to predict: {len(df)}')
 n_batches=math.ceil(len(df)/min_batch_size)
 for batch in trange(n_batches):
start=batch*min_batch_size
end=start+min_batch_size
if batch==n_batches-1:
X=df['question_text'][start:].apply(transformation_func)
 X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')
 y_predicted=model.predict(X)
 else:
X=df['question_text'][start:end].apply(transformation_func)
 X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')
 y_predicted=model.predict(X)
 predictions=np.append(predictions,y_predicted,axis=0)
 return predictions<find_best_params>
categorical_values_train = list(cat_train.index)
 categorical_values_test = list(cat_test.index)
 print(categorical_values_train )
def select_best_threshold(y_true, y_predicted):
thresholds = []
for thresh in np.arange(0.0, 1, 0.001):
thresh = np.round(thresh, 3)
 res=metrics.f1_score(y_true,(y_predicted>thresh ).astype(int))
thresholds.append([thresh, res])
 thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh)
 return best_thresh<train_model>
df_train_add = df_train.copy()
df_test_add = df_test.copy()
lb_make = LabelEncoder()
for i in categorical_values_train:
df_train_add[i] = lb_make.fit_transform(df_train[i])
 for i in categorical_values_test:
df_test_add[i] = lb_make.fit_transform(df_test[i] )
stack_size=3
model_stack={}
t_start=time()
for ind in range(stack_size):
print(f'
=== Model number {ind+1} ===
')
 train_df, val_df=train_val_split(df,frac=0.2)
 model=train_model(train_df,val_df, n_epochs=5)
 y_predicted=get_predictions(val_df,model)
 y_true=val_df['target'].values
best_thresh=select_best_threshold(y_true, y_predicted)
 model_stack[ind]=(model, best_thresh)
 print(f'Model number {ind+1} finished!')
 t_finish=time()
print(f'Total time for training and calculating thresholds is {t_finish-t_start:.2f} seconds' )<load_from_csv>
for i in categorical_values_train:
feature_set = set(df_train[i])
 for j in feature_set:
feature_list = list(feature_set)
 df_train.loc[df_train[i] == j, i] = feature_list.index(j)
 df_train_add.loc[df_train[i] == j, i] = feature_list.index(j)
 for i in categorical_values_test:
feature_set2 = set(df_test[i])
 for j in feature_set2:
feature_list2 = list(feature_set2)
 df_test.loc[df_test[i] == j, i] = feature_list2.index(j)
 df_test_add.loc[df_test[i] == j, i] = feature_list2.index(j )
test_set=pd.read_csv('.. /input/test.csv' )<predict_on_test>
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Lasso
from sklearn.linear_model import Ridge
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import r2_score, mean_squared_error
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import cross_val_score, KFold
predictions={}
for key,(model, threshold)in model_stack.items() :
predictions[key]=(get_predictions(test_set,model)>threshold ).astype(int )<data_type_conversions>
X_train = df_train_add.drop(["SalePrice","LogPrice"], axis=1)
 y_train = df_train_add["LogPrice"]
preds=np.stack([value for value in predictions.values() ],axis=1 ).reshape(-1,stack_size)
 preds=np.average(preds,axis=1)
 preds=np.around(preds,decimals=0 ).astype(int )<prepare_output>
X_training, X_valid, y_training, y_valid = train_test_split(X_train, y_train, test_size=0.2, random_state=0 )
test_set['prediction']=preds
to_submit=test_set[['qid','prediction']]<save_to_csv>
lm = LinearRegression()
lm.fit(X_training,y_training)
 print(lm )
to_submit.to_csv("submission.csv", index=False)
 print('Submissions saved to file!' )<train_model>
predictions = lm.predict(X_valid)
 predictions= predictions.reshape(-1,1 )
print(f'Finishing at {datetime.datetime.now() }' )<load_from_csv>
submission_predictions = np.exp(predictions )
df_train = pd.read_csv('.. /input/train.csv',encoding='utf8')[:10000]
df_train = df_train.fillna('leeg')
 df_test = pd.read_csv('.. /input/test.csv',encoding='utf8')[:50000]
df_test = df_test.fillna('leeg')
 df_train.head(2)
 df_test.head(2)
 end = time.clock()
print('open:',end-start )<choose_model_class>
print('MAE:', metrics.mean_absolute_error(y_valid, submission_predictions))
print('MSE:', metrics.mean_squared_error(y_valid, submission_predictions))
print('RMSE:', np.sqrt(metrics.mean_squared_error(y_valid, submission_predictions)) )
stop = set(stopwords.words('english'))
rake_object = rake.Rake(stop, 3, 3, 1)
 text = "Natural language processing(NLP)deals with the application of computational models to text or speech data.Application areas within NLP include automatic(machine)translation between languages; dialogue systems, which allow a human to interact with a machine using natural language; and information extraction, where the goal is to transform unstructured text into structured(database)representations that can be searched and browsed in flexible ways.NLP technologies are having a dramatic impact on the way people interact with computers, on the way people interact with each other through the use of language, and on the way people access the vast amount of linguistic data now in electronic form.From a scientific viewpoint, NLP involves fundamental questions of how to structure formal models(for example statistical models)of natural language phenomena, and of how to design algorithms that implement these models.In this course you will study mathematical and computational models of language, and the application of these models to key problems in natural language processing.The course has a focus on machine learning methods, which are widely used in modern NLP systems: we will cover formalisms such as hidden Markov models, probabilistic context-free grammars, log-linear models, and statistical models for machine translation.The curriculum closely follows a course currently taught by Professor Collins at Columbia University, and previously taught at MIT."
keywords = rake_object.run(text)
 print("keywords: ", keywords)
<string_transform>
linreg = LinearRegression()
parameters_lin = {"fit_intercept" : [True, False], "normalize" : [True, False], "copy_X" : [True, False]}
grid_linreg = GridSearchCV(linreg, parameters_lin, verbose=1 , scoring = "r2")
 grid_linreg.fit(X_training, y_training)
 print("Best LinReg Model: " + str(grid_linreg.best_estimator_))
print("Best Score: " + str(grid_linreg.best_score_))
def cleantxt(x):
x = str(x)
 for char in ['.', '"', ',', '(', ')', '!', '?', ';', ':']:
x = x.replace(char, ' ')
 return x
def cleantxtsplit(x):
x=cleantxt(x)
 return x.split()
def word_match_share(row):
q1words = {}
q2words = {}
for word in cleantxtsplit(row['question1']):
q1words[word] = 1
for word in cleantxtsplit(row['question2']):
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
nonshared_words_in_q1 = [w for w in q1words.keys() if w not in q2words]
nonshared_words_in_q2 = [w for w in q2words.keys() if w not in q1words]
X1 =(len(shared_words_in_q1)) /(len(q1words))
Y1 =(len(nonshared_words_in_q1)) /(len(q1words))
X2 =(len(shared_words_in_q2)) /(len(q2words))
Y2 =(len(nonshared_words_in_q2)) /(len(q2words))
R3 =(Y1+Y2)/2
return R3
train_word_match = df_train.apply(word_match_share, axis=1, raw=True)
 train_qs = pd.Series(df_train['question1'].map(cleantxt ).tolist() + df_train['question2'].map(cleantxt ).tolist() ).astype(str)
 test_qs = pd.Series(df_test['question1'].map(cleantxt ).tolist() + df_test['question2'].map(cleantxt ).tolist() ).astype(str)
 end = time.clock()
print('clean:',end-start )<init_hyperparams>
linreg = grid_linreg.best_estimator_
linreg.fit(X_training, y_training)
 lin_pred = linreg.predict(X_valid)
 r2_lin = r2_score(y_valid, lin_pred)
 rmse_lin = np.sqrt(mean_squared_error(y_valid, lin_pred))
print("R^2 Score: " + str(r2_lin))
print("RMSE Score: " + str(rmse_lin))
def get_weight(count, eps=10000, min_count=2):
if count < min_count:
return 0
else:
return 1 /(count + eps)
 eps = 5000
words =(" ".join(train_qs)).split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }
end = time.clock()
print('def:',end-start )<feature_engineering>
scores_lin = cross_val_score(linreg, X_training, y_training, cv=10, scoring="r2")
 print("Cross Validation Score: " + str(np.mean(scores_lin)) )
format = lambda x: x.split(' ', 1)[0]
merknamen=df_train['question1'].map(format)
 merknamen=list(set(merknamen))
txt1=df_train['question1']
txt2=df_train['question2']
txt=txt1.append(pd.DataFrame(list(txt2)))
 txt.columns=['q']
print(txt.head())
 tfidf = TfidfVectorizer(ngram_range=(4,4),min_df=1,stop_words=merknamen)
 tfidf.fit(txt.q)
 tfidf_m = tfidf.transform(txt.q)
 words = pd.DataFrame(pd.Series(list(tfidf.vocabulary_.keys()), index=tfidf.vocabulary_.values()),columns=['woord'])
<feature_engineering>
ridge = Ridge()
parameters_ridge = {"fit_intercept" : [True, False], "normalize" : [True, False], "copy_X" : [True, False], "solver" : ["auto"]}
grid_ridge = GridSearchCV(ridge, parameters_ridge, verbose=1, scoring="r2")
 grid_ridge.fit(X_training, y_training)
 print("Best Ridge Model: " + str(grid_ridge.best_estimator_))
print("Best Score: " + str(grid_ridge.best_score_))
def sort_coo(m):
tuples = izip(m.row, m.col, m.data)
 return sorted(tuples, key=lambda x:(x[2]))
txt['core']=''
for xi in range(0,len(txt)) :
rij1=tfidf_m[xi][0:]
print(sort_coo(coo_matrix(rij1)))
 rij1['woord']=words['woord']
toprij=rij1[rij1['plaats']>0]
formathw = lambda x: len(x)
 toprij['lengte']=toprij['woord'].map(formathw)
 toprij=toprij[toprij['lengte']==toprij['lengte'].max() ]['woord']
if len(toprij)>0:
print(list(toprij)[0])
 txt.set_value(xi, 'klas', list(toprij)[0])
 print(txt)
 end = time.clock()
print('tf:',end-start )<sort_values>
ridge = grid_ridge.best_estimator_
ridge.fit(X_training, y_training)
 ridge_pred = ridge.predict(X_valid)
 r2_ridge = r2_score(y_valid, ridge_pred)
 rmse_ridge = np.sqrt(mean_squared_error(y_valid, ridge_pred))
print("R^2 Score: " + str(r2_ridge))
print("RMSE Score: " + str(rmse_ridge))
print('Most common words and weights:
')
 print(sorted(weights.items() , key=lambda x: x[1] if x[1] > 0 else 9999)[:10])
 print('
Least common words and weights: ')
(sorted(weights.items() , key=lambda x: x[1], reverse=True)[:10] )<compute_test_metric>
scores_ridge = cross_val_score(ridge, X_training, y_training, cv=10, scoring="r2")
 print("Cross Validation Score: " + str(np.mean(scores_ridge)) )
def tf(word,blob):
return blob.words.count(word)/len(blob.words)
 def n_containing(word,bloblist):
return blob.words.count(word)/len(blob.words)
 def idf(word,bloblist):
return sum(1 for blob in bloblist if word in blob)
 def tfidf(word,blob,bloblist):
return tf(word,blob)*idf(word,bloblist)
 def tfidf_word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).split() :
q1words[word] = 1
for word in str(row['question2'] ).split() :
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_weights_q1=[weights.get(w, 0)for w in q1words.keys() if w in q2words]
shared_weights_q2=[weights.get(w, 0)for w in q2words.keys() if w in q1words]
nonshared_weights_q1 = [weights.get(w, 0)for w in q1words.keys() if w not in q2words]
nonshared_weights_q2 = [weights.get(w, 0)for w in q2words.keys() if w not in q1words]
total_weights_q1 = [weights.get(w, 0)for w in q1words]
total_weights_q2 =[weights.get(w, 0)for w in q2words]
X1 = np.sum(shared_weights_q1)/ np.sum(total_weights_q1)
 Y1 = np.sum(nonshared_weights_q1)/ np.sum(total_weights_q1)
 X2 = np.sum(shared_weights_q2)/ np.sum(total_weights_q2)
 Y2 = np.sum(nonshared_weights_q2)/ np.sum(total_weights_q2)
 R3 =(Y1+Y2)/2
return R3
end = time.clock()
print('wordmatch:',end-start )<feature_engineering>
from sklearn import ensemble
tfidf_train_word_match = df_train.apply(tfidf_word_match_share, axis=1, raw=True)
 end = time.clock()
print('tfidf:',end-start )<compute_test_metric>
params = {'n_estimators': 20000, 'max_depth': 5, 'min_samples_split': 2,
'learning_rate': 0.05, 'loss': 'ls' , 'max_features' : 20}
clf = ensemble.GradientBoostingRegressor(**params)
 clf.fit(X_training, y_training )
print('Original AUC:', roc_auc_score(df_train['is_duplicate'], train_word_match))
print(' TFIDF AUC:', roc_auc_score(df_train['is_duplicate'], tfidf_train_word_match.fillna(0)) )<prepare_x_and_y>
clf_pred=clf.predict(X_valid)
 clf_pred= clf_pred.reshape(-1,1)
 r2_clf = r2_score(y_valid, clf_pred)
 rmse_clf = np.sqrt(mean_squared_error(y_valid, clf_pred))
print("R^2 Score: " + str(r2_clf))
print("RMSE Score: " + str(rmse_clf))
x_train = pd.DataFrame()
x_test = pd.DataFrame()
x_train['word_match'] = train_word_match
x_train['tfidf_word_match'] = tfidf_train_word_match
x_test['word_match'] = df_test.apply(word_match_share, axis=1, raw=True)
 x_test['tfidf_word_match'] = df_test.apply(tfidf_word_match_share, axis=1, raw=True)
 y_train = df_train['is_duplicate'].values
end = time.clock()
print('createtestdata:',end-start )<concatenate>
scores_clf = cross_val_score(clf, X_training, y_training, cv=10, scoring="r2")
 print("Cross Validation Score: " + str(np.mean(scores_clf)) )
pos_train = x_train[y_train == 1]
neg_train = x_train[y_train == 0]
p = 0.165
scale =(( len(pos_train)/(len(pos_train)+ len(neg_train)))/ p)- 1
while scale > 1:
neg_train = pd.concat([neg_train, neg_train])
 scale -=1
neg_train = pd.concat([neg_train, neg_train[:int(scale * len(neg_train)) ]])
 print(len(pos_train)/(len(pos_train)+ len(neg_train)))
 x_train = pd.concat([pos_train, neg_train])
 y_train =(np.zeros(len(pos_train)) + 1 ).tolist() + np.zeros(len(neg_train)).tolist()
del pos_train, neg_train<split>
dtreg = DecisionTreeRegressor(random_state = 100)
 parameters_dtr = {"criterion" : ["mse", "friedman_mse", "mae"], "splitter" : ["best", "random"], "min_samples_split" : [2, 3, 5, 10],
"max_features" : ["auto", "log2"]}
grid_dtr = GridSearchCV(dtreg, parameters_dtr, verbose=1, scoring="r2")
 grid_dtr.fit(X_training, y_training)
 print("Best DecisionTreeRegressor Model: " + str(grid_dtr.best_estimator_))
print("Best Score: " + str(grid_dtr.best_score_))
x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=4242 )<train_model>
dtr = grid_dtr.best_estimator_
dtreg.fit(X_training, y_training)
 dtr_pred = dtreg.predict(X_valid)
 r2_dtr = r2_score(y_valid, dtr_pred)
 rmse_dtr = np.sqrt(mean_squared_error(y_valid, dtr_pred))
print("R^2 Score: " + str(r2_dtr))
print("RMSE Score: " + str(rmse_dtr))
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
bst = xgb.train(params, d_train, 400, watchlist, early_stopping_rounds=50, verbose_eval=10)
<save_to_csv>
scores_dtr = cross_val_score(dtreg, X_training, y_training, cv=10, scoring="r2")
 print("Cross Validation Score: " + str(np.mean(scores_dtr)) )
d_test = xgb.DMatrix(x_test)
 p_test = bst.predict(d_test)
 sub = pd.DataFrame()
sub['test_id'] = df_test['test_id']
sub['is_duplicate'] = p_test
sub.to_csv('simple_xgb.csv', index=False )<import_modules>
rfr = RandomForestRegressor()
paremeters_rf = {"n_estimators" : [5, 10, 15, 20], "criterion" : ["mse" , "mae"], "min_samples_split" : [2, 3, 5, 10],
"max_features" : ["auto", "log2"]}
grid_rf = GridSearchCV(rfr, paremeters_rf, verbose=1, scoring="r2")
 grid_rf.fit(X_training, y_training)
 print("Best RandomForestRegressor Model: " + str(grid_rf.best_estimator_))
print("Best Score: " + str(grid_rf.best_score_))
import numpy as np<prepare_x_and_y>
rf = grid_rf.best_estimator_
rfr.fit(X_training, y_training)
 rf_pred = rfr.predict(X_valid)
 r2_rf = r2_score(y_valid, rf_pred)
 rmse_rf = np.sqrt(mean_squared_error(y_valid, rf_pred))
print("R^2 Score: " + str(r2_rf))
print("RMSE Score: " + str(rmse_rf))
x = np.array([(62,397,103),(82,347,107),(93,288,120), 
(94,266,128),(65,163,169),(12,102,198), 
(48,138,180),(77,187,157),(85,209,149),(89,316,112)] )<define_variables>
scores_rf = cross_val_score(rfr, X_training, y_training, cv=10, scoring="r2")
 print("Cross Validation Score: " + str(np.mean(scores_rf)) )
x = x - xm<compute_test_metric>
xgboost = XGBRegressor(learning_rate=0.01,n_estimators=20000,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.006)
 xgb = xgboost.fit(X_training, y_training )
v, s, t = np.linalg.svd(x,full_matrices=True )<split>
xgb_pred = xgb.predict(X_valid)
 r2_xgb = r2_score(y_valid, xgb_pred)
 rmse_xgb = np.sqrt(mean_squared_error(y_valid, xgb_pred))
print("R^2 Score: " + str(r2_xgb))
print("RMSE Score: " + str(rmse_xgb))
sigma1 = s[0]
sigma2 = s[1]
sigma3 = s[2]
v1 = t[0]
v2 = t[1]
v3 = t[2]<feature_engineering>
lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=20000,
max_bin=2000,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
 gbm = lightgbm.fit(X_training, y_training )
Z = np.zeros(( x.shape[0],10), np.float32)
 Z[:,0] = x[:,0]**2
Z[:,1] = 2*x[:,0]*x[:,1]
Z[:,2] = 2*x[:,0]*x[:,2]
Z[:,3] = 2*x[:,0]
Z[:,4] = x[:,1]**2
Z[:,5] = 2*x[:,1]*x[:,2]
Z[:,6] = 2*x[:,1]
Z[:,7] = x[:,2]**2
Z[:,8] = 2*x[:,2]
Z[:,9] = 1<feature_engineering>
gbm_pred = gbm.predict(X_valid)
 r2_gbm = r2_score(y_valid, gbm_pred)
 rmse_gbm = np.sqrt(mean_squared_error(y_valid, gbm_pred))
print("R^2 Score: " + str(r2_gbm))
print("RMSE Score: " + str(rmse_gbm))
v, s, t = np.linalg.svd(Z,full_matrices=True)
 smallest_value = np.min(np.array(s))
smallest_index = np.argmin(np.array(s))
T = np.array(t)
 T = T[smallest_index,:]
S = np.zeros(( 4,4),np.float32)
 S[0,0] = T[0]
S[0,1] = S[1,0] = T[1]
S[0,2] = S[2,0] = T[2]
S[0,3] = S[3,0] = T[3]
S[1,1] = T[4]
S[1,2] = S[2,1] = T[5]
S[1,3] = S[3,1] = T[6]
S[2,2] = T[7]
S[2,3] = S[3,2] = T[8]
S[3,3] = T[9]
norm = np.linalg.norm(np.dot(Z,T), ord=2)**2
print(norm )<set_options>
model_performances = pd.DataFrame({
"Model" : ["Linear Regression", "Ridge", "Decision Tree Regressor", "Random Forest Regressor","Gradient Boosting Regression","XGBoost","LGBM Regressor"],
"R Squared" : [str(r2_lin)[0:5], str(r2_ridge)[0:5], str(r2_dtr)[0:5], str(r2_rf)[0:5] , str(r2_clf)[0:5], str(r2_xgb)[0:5], str(r2_gbm)[0:5]],
"RMSE" : [str(rmse_lin)[0:8], str(rmse_ridge)[0:8], str(rmse_dtr)[0:8], str(rmse_rf)[0:8], str(rmse_clf)[0:8], str(rmse_xgb)[0:8], str(rmse_gbm)[0:8]]
})
 model_performances.round(4)
 print("Sorted by R Squared:")
 model_performances.sort_values(by="R Squared", ascending=False )
%matplotlib inline 
<define_variables>
print("Sorted by RMSE:")
 model_performances.sort_values(by="RMSE", ascending=True )
path_to_train = ".. /input/train_1"<define_variables>
def blend_models_predict(X):
return(( 0.05 * lm.predict(X)) + \ 
(0.05 * linreg.predict(X)) + \ 
(0.05 * ridge.predict(X)) + \ 
(0.1 * clf.predict(X)) + \ 
(0.2 * gbm.predict(X)) + \ 
(0.15 * rfr.predict(X)) + \ 
(0.4 * xgb.predict(X)))
event_prefix = "event000001000"<load_from_csv>
submission_predictions = np.exp(blend_models_predict(df_test_add))
print(submission_predictions )
hits, cells, particles, truth = load_event(os.path.join(path_to_train, event_prefix))<categorify>
res=pd.DataFrame(columns = ['Id', 'SalePrice'])
 res['Id'] = df_test.index + 1461
res['SalePrice'] = submission_predictions
res.to_csv('submission1.csv',index=False )