Kaggle competition: House Prices - Advanced Regression Techniques
from spacy.lang.en import English
from spacy.pipeline import DependencyParser
import spacy
from nltk import Tree
from keras.preprocessing import sequence
from keras.preprocessing import text as ktext<compute_test_metric>
def root_mean_squared_log_error(y_valid, y_preds):

 if len(y_preds)!=len(y_valid): return 'error_mismatch'
y_preds_new = [math.log(x)for x in y_preds]
y_valid_new = [math.log(x)for x in y_valid]
return mean_squared_error(y_valid_new, y_preds_new, squared=False )
EMBEDDING_FILE = os.path.join(FAST_TEXT_DATA_FOLDER, 'crawl-300d-2M.vec')
 def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE))<compute_test_metric>
train_data = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 pd.set_option('display.max_columns', None)
 train_data.head()
nlp = spacy.load(spacy_model)
 def bs(list_, target_):
lo, hi = 0, len(list_)-1
while lo < hi:
mid = lo + int(( hi - lo)/ 2)
 if target_ < list_[mid]:
hi = mid
elif target_ > list_[mid]:
lo = mid + 1
else:
return mid + 1
return lo
def bs_(list_, target_):
lo, hi = 0, len(list_)-1
while lo < hi:
mid = lo + int(( hi - lo)/ 2)
 if target_ < list_[mid]:
hi = mid
elif target_ > list_[mid]:
lo = mid + 1
else:
return mid
return lo
def ohe_dist(dist, buckets):
idx = bs_(buckets, dist)
 oh = np.zeros(shape=(len(buckets),), dtype=np.float32)
 oh[idx] = 1
return oh<define_variables>
print(train_data.columns[train_data.isna().any() ].unique())
 len(train_data.columns[train_data.isna().any() ].unique() )
num_pos_features = 45<feature_engineering>
features = [x for x in train_data.columns if x not in ['SalePrice']]
X = train_data.drop(['SalePrice'], axis=1)
 Y = train_data['SalePrice']
def extrac_positional_features(text, char_offset1, char_offset2):
doc = nlp(text)
 max_len = 64
lens = [token.idx for token in doc]
mention_offset1 = bs(lens, char_offset1)- 1
mention_offset2 = bs(lens, char_offset2)- 1
lens = [len(sent)for sent in doc.sents]
acc_lens = [len_ for len_ in lens]
pre_len = 0
for i in range(0, len(acc_lens)) :
pre_len += acc_lens[i]
acc_lens[i] = pre_len
sent_index1 = bs(acc_lens, mention_offset1)
 sent_index2 = bs(acc_lens, mention_offset2)
 sent1 = list(doc.sents)[sent_index1]
sent2 = list(doc.sents)[sent_index2]
bucket_dist = [1, 2, 3, 4, 5, 8, 16, 32, 64]
dist = mention_offset2 - mention_offset1
dist_oh = ohe_dist(dist, bucket_dist)
 bucket_pos = [0, 1, 2, 3, 4, 5, 8, 16, 32]
sent_pos1 = mention_offset1 + 1
if sent_index1 > 0:
sent_pos1 = mention_offset1 - acc_lens[sent_index1-1]
sent_pos_oh1 = ohe_dist(sent_pos1, bucket_pos)
 sent_pos_inv1 = len(sent1)- sent_pos1
assert sent_pos_inv1 >= 0
sent_pos_inv_oh1 = ohe_dist(sent_pos_inv1, bucket_pos)
 sent_pos2 = mention_offset2 + 1
if sent_index2 > 0:
sent_pos2 = mention_offset2 - acc_lens[sent_index2-1]
sent_pos_oh2 = ohe_dist(sent_pos2, bucket_pos)
 sent_pos_inv2 = len(sent2)- sent_pos2
if sent_pos_inv2 < 0:
print(sent_pos_inv2)
 print(len(sent2))
print(sent_pos2)
 raise ValueError
sent_pos_inv_oh2 = ohe_dist(sent_pos_inv2, bucket_pos)
 sent_pos_ratio1 = sent_pos1 / len(sent1)
 sent_pos_ratio2 = sent_pos2 / len(sent2)
 return dist_oh, sent_pos_oh1, sent_pos_oh2, sent_pos_inv_oh1, sent_pos_inv_oh2<concatenate>
X_train, X_valid, y_train, y_valid = train_test_split(X, Y, random_state=42)
 numerical_cols = [cname for cname in X_train.columns if
X_train[cname].dtype in ['int64', 'float64']]
categorical_cols = [cname for cname in X_train.columns if
X_train[cname].nunique() < 13 and
X_train[cname].dtype == "object"]
numerical_transformer = SimpleImputer(strategy='constant')
 categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='most_frequent')) , 
('onehot', OneHotEncoder(handle_unknown='ignore'))
])
 preprocessor = ColumnTransformer(
 transformers=[ 
('num', numerical_transformer, numerical_cols), 
('cat', categorical_transformer, categorical_cols)
 ] )
def create_dist_features(df, text_column, pronoun_offset_column, name_offset_column):
text_offset_list = df[[text_column, pronoun_offset_column, name_offset_column]].values.tolist()
num_features = num_pos_features
pos_feature_matrix = np.zeros(shape=(len(text_offset_list), num_features))
for text_offset_index in range(len(text_offset_list)) :
text_offset = text_offset_list[text_offset_index]
dist_oh, sent_pos_oh1, sent_pos_oh2, sent_pos_inv_oh1, sent_pos_inv_oh2 = extrac_positional_features(text_offset[0], text_offset[1],text_offset[2])
 feature_index = 0
pos_feature_matrix[text_offset_index, feature_index:feature_index+len(dist_oh)] = np.asarray(dist_oh)
 feature_index += len(dist_oh)
 pos_feature_matrix[text_offset_index, feature_index:feature_index+len(sent_pos_oh1)] = np.asarray(sent_pos_oh1)
 feature_index += len(sent_pos_oh1)
 pos_feature_matrix[text_offset_index, feature_index:feature_index+len(sent_pos_oh2)] = np.asarray(sent_pos_oh2)
 feature_index += len(sent_pos_oh2)
 pos_feature_matrix[text_offset_index, feature_index:feature_index+len(sent_pos_inv_oh1)] = np.asarray(sent_pos_inv_oh1)
 feature_index += len(sent_pos_inv_oh1)
 pos_feature_matrix[text_offset_index, feature_index:feature_index+len(sent_pos_inv_oh2)] = np.asarray(sent_pos_inv_oh2)
 feature_index += len(sent_pos_inv_oh2)
 return pos_feature_matrix<define_variables>
random_model = RandomForestRegressor(random_state=42, n_estimators=1000)
 random_clf = Pipeline(steps=[('preprocessor', preprocessor), 
('random_model', random_model)
 ])
 random_clf.fit(X_train, y_train)
 random_clf.fit(X_train, y_train)
 random_preds = random_clf.predict(X_valid)
 print('RMSLE:', root_mean_squared_log_error(y_valid, random_preds))
max_len = 50<string_transform>
xgb_model = XGBRegressor(n_estimators=1000, learning_rate=0.01, random_state=42)
 xgb_clf = Pipeline(steps=[('preprocessor', preprocessor), 
('xgb_model', xgb_model)
 ])
 xgb_clf.fit(X_train, y_train, xgb_model__verbose=False)
 xgb_clf.fit(X_train, y_train)
 xgb_preds = xgb_clf.predict(X_valid)
 print('RMSLE:', root_mean_squared_log_error(y_valid, xgb_preds))
seq_list = list()
def extract_sents(text, char_offset_p, char_offset_a, char_offset_b, id):
global max_len
global seq_list
seq_list.append(list())
 doc = nlp(text)
 token_lens = [token.idx for token in doc]
char_offsets = [char_offset_p, char_offset_a, char_offset_b]
sent_list = list()
for char_offset in char_offsets:
mention_offset = bs(token_lens, char_offset)- 1
mention = doc[mention_offset]
lens = [len(sent)for sent in doc.sents]
acc_lens = [len_ for len_ in lens]
pre_len = 0
for i in range(0, len(acc_lens)) :
pre_len += acc_lens[i]
acc_lens[i] = pre_len
sent_index = bs(acc_lens, mention_offset)
 sent = list(doc.sents)[sent_index]
sent_pos = mention_offset + 1
if sent_index > 0:
sent_pos = mention_offset - acc_lens[sent_index-1]
if len(sent)> max_len:
if sent_pos < max_len-1:
sent_list.append(sent[0:max_len].text)
 sent_list.append(sent_pos)
 seq_list[-1].append(sent[0:max_len])
 else:
sent_list.append(sent[sent_pos-max_len+2 : min(sent_pos+2, len(sent)) ].text)
 sent_list.append(max_len-2)
 seq_list[-1].append(sent[sent_pos-max_len+2 : min(sent_pos+2, len(sent)) ])
 else:
sent_list.append(sent.text)
 sent_list.append(sent_pos)
 seq_list[-1].append(sent)
 return pd.Series([id] + sent_list, index=['ID', 'Pronoun-Sent', 'Pronoun-Sent-Offset', 'A-Sent', 'A-Sent-Offset', 'B-Sent', 'B-Sent-Offset'])
 def add_sent_columns(df, text_column, pronoun_offset_column, a_offset_column, b_offset_column):
global seq_list
seq_list = list()
sent_df = df.apply(lambda row: extract_sents(row.loc[text_column], row[pronoun_offset_column], row[a_offset_column], row[b_offset_column], row['ID']), axis=1)
 df = df.join(sent_df.set_index('ID'), on='ID')
 return df, seq_list<string_transform>
ada_model = AdaBoostRegressor(random_state=42, learning_rate=0.01, n_estimators=1000)
 ada_clf = Pipeline(steps=[('preprocessor', preprocessor), 
('xgb_model', ada_model)
 ])
 ada_clf.fit(X_train, y_train)
 ada_clf.fit(X_train, y_train)
 ada_preds = ada_clf.predict(X_valid)
 print('RMSLE:', root_mean_squared_log_error(y_valid, ada_preds))
seq_list = list()
train_df, train_tokenized = add_sent_columns(train_df, 'Text', 'Pronoun-offset', 'A-offset', 'B-offset')
 seq_list = list()
test_df, test_tokenized = add_sent_columns(test_df, 'Text', 'Pronoun-offset', 'A-offset', 'B-offset')
 seq_list = list()
dev_df, dev_tokenized = add_sent_columns(dev_df, 'Text', 'Pronoun-offset', 'A-offset', 'B-offset')
 seq_list = list()
sub_test_df, sub_test_tokenized = add_sent_columns(sub_test_df, 'Text', 'Pronoun-offset', 'A-offset', 'B-offset')
 train_tokenized = train_tokenized[1:]
test_tokenized = test_tokenized[1:]
dev_tokenized = dev_tokenized[1:]
sub_test_tokenized = sub_test_tokenized[1:]<define_variables>
train_data['OverallQual'].isnull().sum()
embed_size = 300
max_features = 80000
word_index = dict()
idx = 1
for text_ in train_tokenized+test_tokenized+dev_tokenized+sub_test_tokenized:
for sent_ in text_:
for word_ in sent_:
if word_.text not in word_index and word_.text in embeddings_index:
word_index[word_.text] = idx
idx += 1
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words + 1, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = None
if word in embeddings_index:
embedding_vector = embeddings_index[word]
if embedding_vector is not None: embedding_matrix[i] = embedding_vector
print(embedding_matrix.shape)
 pos_index = dict()
idx = 1
for text_ in train_tokenized+test_tokenized+dev_tokenized+sub_test_tokenized:
for sent_ in text_:
for word_ in sent_:
if word_.pos not in pos_index:
pos_index[word_.pos] = idx
idx += 1
def sentences_to_sequences(tokenized_):
return list(map(
 lambda sent_tokenized: list(map(
 lambda token_: word_index[token_.text] if token_.text in word_index else 0,
sent_tokenized 
)) ,
tokenized_ 
))
def poses_to_sequences(tokenized_):
return list(map(
 lambda sent_tokenized: list(map(
 lambda token_: pos_index[token_.pos] if token_.pos in pos_index else 0,
sent_tokenized 
)) ,
tokenized_ 
)) 
<string_transform>
train_data['OverallCond'].isnull().sum()
train_p_tokenized = sentences_to_sequences([row[0] for row in train_tokenized])
 train_a_tokenized = sentences_to_sequences([row[1] for row in train_tokenized])
 train_b_tokenized = sentences_to_sequences([row[2] for row in train_tokenized])
 test_p_tokenized = sentences_to_sequences([row[0] for row in test_tokenized])
 test_a_tokenized = sentences_to_sequences([row[1] for row in test_tokenized])
 test_b_tokenized = sentences_to_sequences([row[2] for row in test_tokenized])
 sub_test_p_tokenized = sentences_to_sequences([row[0] for row in sub_test_tokenized])
 sub_test_a_tokenized = sentences_to_sequences([row[1] for row in sub_test_tokenized])
 sub_test_b_tokenized = sentences_to_sequences([row[2] for row in sub_test_tokenized])
 dev_p_tokenized = sentences_to_sequences([row[0] for row in dev_tokenized])
 dev_a_tokenized = sentences_to_sequences([row[1] for row in dev_tokenized])
 dev_b_tokenized = sentences_to_sequences([row[2] for row in dev_tokenized])
 seq_p_train = sequence.pad_sequences(train_p_tokenized, maxlen = max_len, padding='post')
 seq_a_train = sequence.pad_sequences(train_a_tokenized, maxlen = max_len, padding='post')
 seq_b_train = sequence.pad_sequences(train_b_tokenized, maxlen = max_len, padding='post')
 seq_p_test = sequence.pad_sequences(test_p_tokenized, maxlen = max_len, padding='post')
 seq_a_test = sequence.pad_sequences(test_a_tokenized, maxlen = max_len, padding='post')
 seq_b_test = sequence.pad_sequences(test_b_tokenized, maxlen = max_len, padding='post')
 seq_p_sub_test = sequence.pad_sequences(sub_test_p_tokenized, maxlen = max_len, padding='post')
 seq_a_sub_test = sequence.pad_sequences(sub_test_a_tokenized, maxlen = max_len, padding='post')
 seq_b_sub_test = sequence.pad_sequences(sub_test_b_tokenized, maxlen = max_len, padding='post')
 seq_p_dev = sequence.pad_sequences(dev_p_tokenized, maxlen = max_len, padding='post')
 seq_a_dev = sequence.pad_sequences(dev_a_tokenized, maxlen = max_len, padding='post')
 seq_b_dev = sequence.pad_sequences(dev_b_tokenized, maxlen = max_len, padding='post')
 train_p_pos = poses_to_sequences([row[0] for row in train_tokenized])
 train_a_pos = poses_to_sequences([row[1] for row in train_tokenized])
 train_b_pos = poses_to_sequences([row[2] for row in train_tokenized])
 test_p_pos = poses_to_sequences([row[0] for row in test_tokenized])
 test_a_pos = poses_to_sequences([row[1] for row in test_tokenized])
 test_b_pos = poses_to_sequences([row[2] for row in test_tokenized])
 sub_test_p_pos = poses_to_sequences([row[0] for row in sub_test_tokenized])
 sub_test_a_pos = poses_to_sequences([row[1] for row in sub_test_tokenized])
 sub_test_b_pos = poses_to_sequences([row[2] for row in sub_test_tokenized])
 dev_p_pos = poses_to_sequences([row[0] for row in dev_tokenized])
 dev_a_pos = poses_to_sequences([row[1] for row in dev_tokenized])
 dev_b_pos = poses_to_sequences([row[2] for row in dev_tokenized])
 pos_p_train = sequence.pad_sequences(train_p_pos, maxlen = max_len, padding='post')
 pos_a_train = sequence.pad_sequences(train_a_pos, maxlen = max_len, padding='post')
 pos_b_train = sequence.pad_sequences(train_b_pos, maxlen = max_len, padding='post')
 pos_p_test = sequence.pad_sequences(test_p_pos, maxlen = max_len, padding='post')
 pos_a_test = sequence.pad_sequences(test_a_pos, maxlen = max_len, padding='post')
 pos_b_test = sequence.pad_sequences(test_b_pos, maxlen = max_len, padding='post')
 pos_p_sub_test = sequence.pad_sequences(sub_test_p_pos, maxlen = max_len, padding='post')
 pos_a_sub_test = sequence.pad_sequences(sub_test_a_pos, maxlen = max_len, padding='post')
 pos_b_sub_test = sequence.pad_sequences(sub_test_b_pos, maxlen = max_len, padding='post')
 pos_p_dev = sequence.pad_sequences(dev_p_pos, maxlen = max_len, padding='post')
 pos_a_dev = sequence.pad_sequences(dev_a_pos, maxlen = max_len, padding='post')
 pos_b_dev = sequence.pad_sequences(dev_b_pos, maxlen = max_len, padding='post')
 index_p_train = train_df['Pronoun-Sent-Offset'].values
index_a_train = train_df['A-Sent-Offset'].values
index_b_train = train_df['B-Sent-Offset'].values
index_p_test = test_df['Pronoun-Sent-Offset'].values
index_a_test = test_df['A-Sent-Offset'].values
index_b_test = test_df['B-Sent-Offset'].values
index_p_sub_test = sub_test_df['Pronoun-Sent-Offset'].values
index_a_sub_test = sub_test_df['A-Sent-Offset'].values
index_b_sub_test = sub_test_df['B-Sent-Offset'].values
index_p_dev = dev_df['Pronoun-Sent-Offset'].values
index_a_dev = dev_df['A-Sent-Offset'].values
index_b_dev = dev_df['B-Sent-Offset'].values
pa_pos_tra = create_dist_features(train_df, 'Text', 'Pronoun-offset', 'A-offset')
 pa_pos_dev = create_dist_features(dev_df, 'Text', 'Pronoun-offset', 'A-offset')
 pa_pos_test = create_dist_features(test_df, 'Text', 'Pronoun-offset', 'A-offset')
 pa_pos_sub_test = create_dist_features(sub_test_df, 'Text', 'Pronoun-offset', 'A-offset')
 pb_pos_tra = create_dist_features(train_df, 'Text', 'Pronoun-offset', 'B-offset')
 pb_pos_dev = create_dist_features(dev_df, 'Text', 'Pronoun-offset', 'B-offset')
 pb_pos_test = create_dist_features(test_df, 'Text', 'Pronoun-offset', 'B-offset')
 pb_pos_sub_test = create_dist_features(sub_test_df, 'Text', 'Pronoun-offset', 'B-offset' )<prepare_x_and_y>
train_data['YearBuilt'].isnull().sum()
X_train = [seq_p_train, seq_a_train, seq_b_train, pos_p_train, pos_a_train, pos_b_train, index_p_train, index_a_train, index_b_train, pa_pos_tra, pb_pos_tra]
X_dev = [seq_p_dev, seq_a_dev, seq_b_dev, pos_p_dev, pos_a_dev, pos_b_dev, index_p_dev, index_a_dev, index_b_dev, pa_pos_dev, pb_pos_dev]
X_test = [seq_p_test, seq_a_test, seq_b_test, pos_p_test, pos_a_test, pos_b_test, index_p_test, index_a_test, index_b_test, pa_pos_test, pb_pos_test]
X_sub_test = [seq_p_sub_test, seq_a_sub_test, seq_b_sub_test, pos_p_sub_test, pos_a_sub_test, pos_b_sub_test, index_p_sub_test, index_a_sub_test, index_b_sub_test, pa_pos_sub_test, pb_pos_sub_test]<prepare_x_and_y>
train_data['YearRemodAdd'].isnull().sum()
def _row_to_y(row):
if row.loc['A-coref']:
return 0
if row.loc['B-coref']:
return 1
return 2
y_tra = train_df.apply(_row_to_y, axis=1)
 y_dev = dev_df.apply(_row_to_y, axis=1)
 y_test = test_df.apply(_row_to_y, axis=1 )<import_modules>
train_data['TotalBsmtSF'].isnull().sum()
import numpy as np
from keras import backend
from keras import layers
from keras import models
from keras import initializers, regularizers, constraints, activations
from keras.engine import Layer
import keras.backend as K
from keras.layers import merge
import tensorflow as tf<concatenate>
Q1 = train_data['TotalBsmtSF'].quantile(0.25)
 Q3 = train_data['TotalBsmtSF'].quantile(0.75)
 IQR = Q3 - Q1
outliers = train_data.loc[(train_data['TotalBsmtSF'] >(Q3 + 1.75 * IQR)) |(train_data['TotalBsmtSF'] <(Q1 - 1.75 * IQR)) , 'TotalBsmtSF']
print("Percent of Outliers: ", outliers.count() / train_data['TotalBsmtSF'].count() * 100 )
def _dot_product(x, kernel):

 if K.backend() == 'tensorflow':
return K.squeeze(K.dot(x, K.expand_dims(kernel)) , axis=-1)
 else:
return K.dot(x, kernel)
 class AttentionWeight(Layer):

 def __init__(self,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
super(AttentionWeight, self ).__init__(**kwargs)
 def build(self, input_shape):
shape1 = input_shape[0]
shape2 = input_shape[1]
self.W = self.add_weight(( shape2[-1], shape1[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 if self.bias:
self.b = self.add_weight(( shape2[-1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 def compute_mask(self, input, input_mask=None):
return None
def call(self, inputs, mask=None):
x = inputs[0]
u = inputs[1]
uit = _dot_product(x, self.W)
 if self.bias:
uit += self.b
uit = K.tanh(uit)
 ait = K.batch_dot(uit, u)
 a = K.exp(ait)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 return a
def compute_output_shape(self, input_shape):
if not isinstance(input_shape, list)or len(input_shape)!= 2:
raise ValueError('A `Dot` layer should be called '
'on a list of 2 inputs.')
 shape1 = list(input_shape[0])
 shape2 = list(input_shape[1])
 return shape1[0], shape1[1]
def get_config(self):
config = {
'W_regularizer': regularizers.serialize(self.W_regularizer),
'b_regularizer': regularizers.serialize(self.b_regularizer),
'W_constraint': constraints.serialize(self.W_constraint),
'b_constraint': constraints.serialize(self.b_constraint),
'bias': self.bias
}
base_config = super(AttentionWeight, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
class FeatureSelection1D(Layer):

 def __init__(self, num_selects, **kwargs):
self.num_selects = num_selects
self.supports_masking = True
super(FeatureSelection1D, self ).__init__(**kwargs)
 def build(self, input_shape):
super(FeatureSelection1D, self ).build(input_shape)
 def compute_mask(self, input, input_mask=None):
return None
def call(self, inputs, mask=None):
if not isinstance(inputs, list)or len(inputs)!= 2:
raise ValueError('FeatureSelection1D layer should be called '
'on a list of 2 inputs.')
 if mask is not None:
a = K.cast(mask, K.floatx())* inputs[0]
else:
a = inputs[0]
b = inputs[1]
a = tf.batch_gather(a, b)
 return a
def compute_output_shape(self, input_shape):
if not isinstance(input_shape, list)or len(input_shape)!= 2:
raise ValueError('A `FeatureSelection1D` layer should be called '
'on a list of 2 inputs.')
 shape1 = list(input_shape[0])
 shape2 = list(input_shape[1])
 if shape2[0] != shape1[0]:
raise ValueError("batch size must be same")
 if shape2[1] != self.num_selects:
raise ValueError("must conform to the num_select")
 return(shape1[0], self.num_selects, shape1[2])
 def get_config(self):
config = {
'num_selects': self.num_selects
}
base_config = super(FeatureSelection1D, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<define_variables>
train_data.drop(train_data.loc[(train_data['TotalBsmtSF'] >(Q3 + 1.75 * IQR)) |(train_data['TotalBsmtSF'] <(Q1 - 1.75 * IQR)) ].index, inplace=True)
 train_data.shape
histories = list()
file_paths = list()
cos = list()<choose_model_class>
train_data['1stFlrSF'].isnull().sum()
def build_e2e_birnn_attention_model(
 voca_dim, time_steps, pos_tag_size, pos_tag_dim, extra_feature_dims, output_dim, rnn_dim, model_dim, mlp_dim,
item_embedding=None, rnn_depth=1, mlp_depth=1,
drop_out=0.5, rnn_drop_out=0., rnn_state_drop_out=0.,
trainable_embedding=False, gpu=False, return_customized_layers=False):

 if item_embedding is not None:
inputp = models.Input(shape=(time_steps,), dtype='int32', name='inputp')
 inputa = models.Input(shape=(time_steps,), dtype='int32', name='inputa')
 inputb = models.Input(shape=(time_steps,), dtype='int32', name='inputb')
 inputs = [inputp, inputa, inputb]
if isinstance(item_embedding, np.ndarray):
assert voca_dim == item_embedding.shape[0]
embed_dim = item_embedding.shape[1]
emb_layer = layers.Embedding(
 voca_dim, item_embedding.shape[1], input_length=time_steps,
weights=[item_embedding, ], trainable=trainable_embedding,
mask_zero=False, name='embedding_layer0' 
)
 elif utils.is_integer(item_embedding):
embed_dim = item_embedding
emb_layer = layers.Embedding(
 voca_dim, item_embedding, input_length=time_steps,
trainable=trainable_embedding,
mask_zero=False, name='embedding_layer0' 
)
 else:
raise ValueError("item_embedding must be either integer or numpy matrix")
 xs = list(map(
 lambda input_: emb_layer(input_),
inputs 
))
else:
inputp = models.Input(shape=(time_steps, voca_dim), dtype='float32', name='inputp')
 inputa = models.Input(shape=(time_steps, voca_dim), dtype='float32', name='inputa')
 inputb = models.Input(shape=(time_steps, voca_dim), dtype='float32', name='inputb')
 embed_dim = voca_dim
xs = [inputp, inputa, inputb]
inputposp = models.Input(shape=(time_steps,), dtype='int32', name='inputposp')
 inputposa = models.Input(shape=(time_steps,), dtype='int32', name='inputposa')
 inputposb = models.Input(shape=(time_steps,), dtype='int32', name='inputposb')
 inputpos = [inputposp, inputposa, inputposb]
pos_emb_layer = layers.Embedding(
 pos_tag_size, pos_tag_dim, input_length=time_steps,
trainable=True, mask_zero=False, name='pos_embedding_layer0' 
)
 xpos = list(map(
 lambda input_: pos_emb_layer(input_),
inputpos 
))
embed_concate_layer = layers.Concatenate(axis=2, name="embed_concate_layer")
 for i in range(len(xs)) :
xs[i] = embed_concate_layer([xs[i], xpos[i]])
 inputpi = models.Input(shape=(1,), dtype='int32', name='inputpi')
 inputai = models.Input(shape=(1,), dtype='int32', name='inputai')
 inputbi = models.Input(shape=(1,), dtype='int32', name='inputbi')
 xis = [inputpi, inputai, inputbi]
birnns = list()
rnn_batchnorms = list()
rnn_dropouts = list()
if gpu:
for i in range(rnn_depth):
rnn_dropout = layers.SpatialDropout1D(rnn_drop_out)
 birnn = layers.Bidirectional(
 layers.CuDNNGRU(rnn_dim, return_sequences=True),
name='bi_lstm_layer' + str(i))
rnn_batchnorm = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))
birnns.append(birnn)
 rnn_dropouts.append(rnn_dropout)
 rnn_batchnorms.append(rnn_batchnorm)
 xs_ = list()
for x_ in xs:
for i in range(len(birnns)) :
x_ = rnn_dropouts[i](x_)
 x_ = birnns[i](x_)
 x_ = rnn_batchnorms[i](x_)
 xs_.append(x_)
 xs = xs_
else:
for i in range(rnn_depth):
birnn = layers.Bidirectional(
 layers.GRU(rnn_dim, return_sequences=True, dropout=rnn_drop_out,
recurrent_dropout=rnn_state_drop_out),
name='bi_lstm_layer' + str(i))
rnn_batchnorm = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))
birnns.append(birnn)
 rnn_batchnorms.append(rnn_batchnorm)
 xs_ = list()
for x_ in xs:
for i in range(len(birnns)) :
x_ = birnns[i](x_)
 x_ = rnn_batchnorms[i](x_)
 xs_.append(x_)
 xs = xs_
select_layer = FeatureSelection1D(1, name='boundary_selection_layer')
 flatten_layer1 = layers.Flatten('channels_first', name="flatten_layer1")
 permute_layer = layers.Permute(( 2, 1), name='permuted_attention_x')
 attent_weight = AttentionWeight(name="attention_weight")
 focus_layer = layers.Dot([2, 1], name='focus' + '_layer')
 reshape_layer = layers.Reshape(( 1, rnn_dim*2), name="reshape_layer")
 concate_layer = layers.Concatenate(axis=1, name="attention_concate_layer")
 atten_dropout_layer = layers.Dropout(drop_out, name='attention_dropout_layer')
 map_layer1 = layers.Dense(model_dim, activation="relu", name="map_layer1")
 map_layer2 = map_layer1
flatten_layer = layers.Flatten('channels_first', name="flatten_layer")
 for i in range(len(xs)) :
if i == 0:
map_layer = map_layer1
else:
map_layer = map_layer2
select_ = select_layer([xs[i], xis[i]])
 flatten_select_ = flatten_layer1(select_)
 att = attent_weight([xs[i], flatten_select_])
 focus = focus_layer([permute_layer(xs[i]), att])
 xs[i] = concate_layer([select_, reshape_layer(focus)])
 xs[i] = flatten_layer(xs[i])
 xs[i] = atten_dropout_layer(xs[i])
 xs[i] = map_layer(xs[i])
 x = layers.Concatenate(axis=1, name="concat_feature_layer" )(xs)
 x = layers.Dropout(drop_out, name='dropout_layer' )(x)
 for i in range(mlp_depth - 1):
x = layers.Dense(mlp_dim, activation='selu', kernel_initializer='lecun_normal', name='selu_layer' + str(i))(x)
 x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x)
 outputs = layers.Dense(output_dim, activation="softmax", name="softmax_layer0" )(x)
 model = models.Model([inputp, inputa, inputb] + inputpos + xis , outputs)
 if return_customized_layers:
return model, {'FeatureSelection1D': FeatureSelection1D, 'AttentionWeight': AttentionWeight}
return model<choose_model_class>
Q1 = train_data['1stFlrSF'].quantile(0.25)
 Q3 = train_data['1stFlrSF'].quantile(0.75)
 IQR = Q3 - Q1
outliers = train_data.loc[(train_data['1stFlrSF'] >(Q3 + 1.75 * IQR)) |(train_data['1stFlrSF'] <(Q1 - 1.75 * IQR)) , '1stFlrSF']
print("Percent of Outliers: ", outliers.count() / train_data['1stFlrSF'].count() * 100 )
voca_dim = embedding_matrix.shape[0]
pos_tag_size = len(pos_index)
 time_steps = max_len
embed_dim = embedding_matrix.shape[1]
pos_tag_dim = 5
extra_feature_dims = num_pos_features
output_dim = 3
rnn_dim = 50
model_dim = 10
mlp_dim = 10
rnn_depth = 1
mlp_depth=1
drop_out=0.2
rnn_drop_out=0.5
gpu = True
return_customized_layers=True
model, co = build_e2e_birnn_attention_model(
 voca_dim, time_steps, pos_tag_size, pos_tag_dim, extra_feature_dims, output_dim, rnn_dim, model_dim, mlp_dim,
item_embedding=embedding_matrix, rnn_depth=rnn_depth, mlp_depth=mlp_depth,
drop_out=drop_out, rnn_drop_out=rnn_drop_out, rnn_state_drop_out=rnn_drop_out,
trainable_embedding=False, gpu=gpu, return_customized_layers=return_customized_layers)
 cos.append(co )<set_options>
train_data.drop(train_data.loc[(train_data['1stFlrSF'] >(Q3 + 1.75 * IQR)) |(train_data['1stFlrSF'] <(Q1 - 1.75 * IQR)) ].index, inplace=True)
 train_data.shape
img = Image.open('model.png')
 display(img )<train_on_grid>
train_data['GrLivArea'].isnull().sum()
adam = ko.Nadam(clipnorm=1.0)
 model.compile(adam, loss="sparse_categorical_crossentropy", metrics=["sparse_categorical_accuracy"])
 file_path = "best_e2e_rnn_atten_model.hdf5"
check_point = kc.ModelCheckpoint(file_path, monitor = "val_loss", verbose = 1, save_best_only = True, mode = "min")
 history = model.fit(X_train, y_tra, batch_size=30, epochs=40, validation_data=(X_dev, y_dev), callbacks = [check_point])
 file_paths.append(file_path)
 histories.append(np.min(np.asarray(history.history['val_loss'])))
 del model, history
gc.collect()<load_pretrained>
Q1 = train_data['GrLivArea'].quantile(0.25)
 Q3 = train_data['GrLivArea'].quantile(0.75)
 IQR = Q3 - Q1
outliers = train_data.loc[(train_data['GrLivArea'] >(Q3 + 1.75 * IQR)) |(train_data['GrLivArea'] <(Q1 - 1.75 * IQR)) , 'GrLivArea']
print("Percent of Outliers: ", outliers.count() / train_data['GrLivArea'].count() * 100 )
print("load best model: " + str(file_paths[np.argmin(histories)]))
model = models.load_model(
 file_paths[np.argmin(histories)], cos[np.argmin(histories)] )<compute_test_metric>
train_data['FullBath'].isnull().sum()
loss, score = model.evaluate(x=X_test, y=y_test)
 print('model evalutation loss = {} and score = {}'.format(loss, score))<predict_on_test>
train_data['TotRmsAbvGrd'].isnull().sum()
y_sub_preds = model.predict(X_sub_test, batch_size = 1024, verbose = 1)
 sub_df_path = os.path.join(SUB_DATA_FOLDER, 'sample_submission_stage_2.csv')
 sub_df = pd.read_csv(sub_df_path)
 sub_df.loc[:, 'A'] = pd.Series(y_sub_preds[:, 0])
 sub_df.loc[:, 'B'] = pd.Series(y_sub_preds[:, 1])
 sub_df.loc[:, 'NEITHER'] = pd.Series(y_sub_preds[:, 2])
 sub_df.head()<save_to_csv>
train_data['GarageCars'].isnull().sum()
sub_df.to_csv("submission.csv", index=False )<load_from_csv>
train_data['GarageArea'].isnull().sum()
nlp = spacy.load('en_core_web_sm')
 test = pd.read_csv('.. /input/test_stage_2.tsv', delimiter='\t' ).rename(columns={'A': 'A_Noun', 'B': 'B_Noun'})
 sub = pd.read_csv('.. /input/sample_submission_stage_2.csv')
 test.shape, sub.shape<load_from_csv>
Q1 = train_data['GarageArea'].quantile(0.25)
 Q3 = train_data['GarageArea'].quantile(0.75)
 IQR = Q3 - Q1
outliers = train_data.loc[(train_data['GarageArea'] >(Q3 + 1.75 * IQR)) |(train_data['GarageArea'] <(Q1 - 1.75 * IQR)) , 'GarageArea']
print("Percent of Outliers: ", outliers.count() / train_data['GarageArea'].count() * 100 )
gh_test = pd.read_csv("https://raw.githubusercontent.com/google-research-datasets/gap-coreference/master/gap-test.tsv", delimiter='\t')
 gh_train = pd.read_csv("https://raw.githubusercontent.com/google-research-datasets/gap-coreference/master/gap-development.tsv", delimiter='\t')
 gh_valid = pd.read_csv("https://raw.githubusercontent.com/google-research-datasets/gap-coreference/master/gap-validation.tsv", delimiter='\t')
 train = pd.concat(( gh_test, gh_train, gh_valid)).rename(columns={'A': 'A_Noun', 'B': 'B_Noun'} ).reset_index(drop=True)
 train.shape<feature_engineering>
train_data.drop(train_data.loc[(train_data['GarageArea'] >(Q3 + 1.75 * IQR)) |(train_data['GarageArea'] <(Q1 - 1.75 * IQR)) ].index, inplace=True)
 train_data.shape
def name_replace(s, r1, r2):
s = str(s ).replace(r1,r2)
 for r3 in r1.split(' '):
s = str(s ).replace(r3,r2)
 return s
def get_features(df):
df['section_min'] = df[['Pronoun-offset', 'A-offset', 'B-offset']].min(axis=1)
 df['Pronoun-offset2'] = df['Pronoun-offset'] + df['Pronoun'].map(len)
 df['A-offset2'] = df['A-offset'] + df['A_Noun'].map(len)
 df['B-offset2'] = df['B-offset'] + df['B_Noun'].map(len)
 df['A-dist_abs'] =(df['Pronoun-offset'] - df['A-offset'] ).abs()
df['B-dist_abs'] =(df['Pronoun-offset'] - df['B-offset'] ).abs()
df['A-dist'] =(df['Pronoun-offset'] - df['A-offset'])
 df['B-dist'] =(df['Pronoun-offset'] - df['B-offset'])
 df['section_max'] = df[['Pronoun-offset2', 'A-offset2', 'B-offset2']].max(axis=1)
 df['A_max'] =(df['A-offset2'] == df['section_max'] ).astype(int)
 df['A_min'] =(df['A-offset2'] == df['section_min'] ).astype(int)
 df['B_max'] =(df['B-offset2'] == df['section_max'] ).astype(int)
 df['B_min'] =(df['B-offset2'] == df['section_min'] ).astype(int)
 df['wc'] = df.apply(lambda r: len(str(r['Text'][r['section_min']: r['section_max']] ).split(' ')) , axis=1)
 df['Text'] = df.apply(lambda r: name_replace(r['Text'], r['A_Noun'], 'subjectone'), axis=1)
 df['Text'] = df.apply(lambda r: name_replace(r['Text'], r['B_Noun'], 'subjecttwo'), axis=1)
 return(df)
 train = get_features(train)
 test = get_features(test )<feature_engineering>
def get_nlp_features(s, w):
doc = nlp(str(s))
tokens = pd.DataFrame([[token.text, token.dep_] for token in doc], columns=['text', 'dep'])
 return len(tokens[(( tokens['text']==w)&(tokens['dep']=='poss')) ])
 train['A-poss'] = train['Text'].map(lambda x: get_nlp_features(x, 'subjectone'))
train['B-poss'] = train['Text'].map(lambda x: get_nlp_features(x, 'subjecttwo'))
test['A-poss'] = test['Text'].map(lambda x: get_nlp_features(x, 'subjectone'))
test['B-poss'] = test['Text'].map(lambda x: get_nlp_features(x, 'subjecttwo'))<data_type_conversions>
hp_model = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0.5, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.02, max_delta_step=0, max_depth=4,
min_child_weight=1, monotone_constraints='() ',
n_estimators=1000, n_jobs=0, num_parallel_tree=1, random_state=42,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=0.8,
tree_method='exact', validate_parameters=1, verbosity=None)
 hp_clf = Pipeline(steps=[('preprocessor', preprocessor), 
('hp_model', hp_model)
 ])
 hp_clf.fit(X_train, y_train, hp_model__verbose=False)
 hp_preds = hp_clf.predict(X_valid)
 print('RMSLE:', root_mean_squared_log_error(y_valid, hp_preds))
train = train.rename(columns={'A-coref':'A', 'B-coref':'B'})
 train['A'] = train['A'].astype(int)
 train['B'] = train['B'].astype(int)
 train['NEITHER'] = 1.0 -(train['A'] + train['B'] )<train_model>
X = train_data.drop(['SalePrice'], axis=1)
 y = train_data.SalePrice
col = ['Pronoun-offset', 'A-offset', 'B-offset', 'section_min', 'Pronoun-offset2', 'A-offset2', 'B-offset2', 'section_max',
'A-poss', 'B-poss', 'A-dist', 'B-dist', 'A-dist_abs', 'B-dist_abs', 'A_max', 'A_min', 'B_max', 'B_min', 'wc']
x1, x2, y1, y2 = model_selection.train_test_split(train[col].fillna(-1), train[['A', 'B', 'NEITHER']], test_size=0.2, random_state=123)
 etc = multiclass.OneVsRestClassifier(ensemble.ExtraTreesClassifier(n_jobs=-1, max_depth=9, n_estimators=1000, random_state=123))
etc.fit(x1, y1)
 print('log_loss', metrics.log_loss(y2, etc.predict_proba(x2)))
 etc.fit(train[col].fillna(-1), train[['A', 'B', 'NEITHER']])
 results = etc.predict_proba(test[col])
 test['A'] = results[:,0]
test['B'] = results[:,1]
test['NEITHER'] = results[:,2]
test[['ID', 'A', 'B', 'NEITHER']].to_csv('submission.csv', index=False )<load_pretrained>
X.columns.to_list()
x = np.load('.. /input/prepare-for-submission/0.npy' )<choose_model_class>
X_feat_eng = X.copy()
X_feat_eng['years_since_update'] = X_feat_eng['YearRemodAdd'] - X_feat_eng['YearBuilt']
X_feat_eng['geometry'] = X_feat_eng['LotArea'] / X_feat_eng['LotFrontage']
X_feat_eng['land_topology'] = X_feat_eng['LandSlope'] + '_' + X_feat_eng['LandContour']
X_feat_eng['value_proposition'] = X_feat_eng['YearBuilt'] * X_feat_eng['OverallQual']
X_feat_eng['finished_basement'] = X_feat_eng['BsmtFinSF1'] > 0
X_feat_eng['garage_value'] = X_feat_eng['YearBuilt'] * X_feat_eng['GarageCars']
X_feat_eng['misc_value'] = X_feat_eng['Fireplaces'] + X_feat_eng['OverallQual']
X_feat_eng = X_feat_eng.drop(columns=['GarageCars'])
 X_feat_eng['Test'] = X_feat_eng['Street'] + '_' + X_feat_eng['Alley']
feature_numerical_cols = [cname for cname in X_feat_eng.columns if
X_feat_eng[cname].dtype in ['int64', 'float64']]
feature_categorical_cols = [cname for cname in X_feat_eng.columns if X_feat_eng[cname].nunique() < 50
and X_feat_eng[cname].dtype in ['object', 'bool']]
feature_numerical_transformer = SimpleImputer(strategy='constant')
 feature_categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='most_frequent')) , 
('onehot', OneHotEncoder(handle_unknown='ignore'))
])
 feature_preprocessor = ColumnTransformer(
 transformers=[ 
('num', feature_numerical_transformer, feature_numerical_cols), 
('cat', feature_categorical_transformer, feature_categorical_cols)
 ])
 feature_model = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0.0, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.02, max_delta_step=0, max_depth=4,
min_child_weight=0.0, monotone_constraints='() ',
n_estimators=1250, n_jobs=0, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=0.8,
tree_method='exact', validate_parameters=1, verbosity=None)
 feature_clf = Pipeline(steps=[('feature_preprocessor', feature_preprocessor), 
('feature_model', feature_model)
 ])
 feature_X_train, feature_X_valid, feature_y_train, feature_y_valid = train_test_split(X_feat_eng, y, random_state=0)
 feature_clf.fit(feature_X_train, feature_y_train, feature_model__verbose=False)
 feature_preds = feature_clf.predict(feature_X_valid)
 print('RMSLE:', root_mean_squared_log_error(feature_y_valid, feature_preds))
def logloss(y, y_):
return tf.losses.log_loss(y,y_)
 model = load_model(mf, custom_objects={'logloss':logloss} )<import_modules>
X_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
import gc<predict_on_test>
X_test['years_since_update'] = X_test['YearRemodAdd'] - X_test['YearBuilt']
X_test['geometry'] = X_test['LotArea'] / X_test['LotFrontage']
X_test['land_topology'] = X_test['LandSlope'] + '_' + X_test['LandContour']
X_test['value_proposition'] = X_test['YearBuilt'] * X_test['OverallQual']
X_test['finished_basement'] = X_test['BsmtFinSF1'] > 0
X_test['garage_value'] = X_test['YearBuilt'] * X_test['GarageCars']
X_test['misc_value'] = X_test['Fireplaces'] + X_test['OverallQual']
X_test['Test'] = X_test['Street'] + '_' + X_test['Alley']
X_test = X_test.drop(columns=['GarageCars'])
 feature_clf.fit(X_feat_eng, y, feature_model__verbose=False )
y1 = model.predict(x )<set_options>
preds = feature_clf.predict(X_test)
 output = pd.DataFrame({'Id': X_test.Id,
'SalePrice': preds})
 output.to_csv('submission.csv', index=False )