Kaggle competition: House Price Prediction
X = df["question_text"].fillna("_
df = add_features(df)
 features = df[['caps_vs_length', 'words_vs_unique']].fillna(0)
 ss = StandardScaler()
ss.fit(features)
 features = ss.transform(features)
 tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X))
X = tokenizer.texts_to_sequences(X)
 X = pad_sequences(X, maxlen=maxlen )<split>
numeric_features_train = list(train.select_dtypes(exclude =
['object', 'category'] ).columns)
 numeric_features_test = list(test.select_dtypes(exclude =
['object', 'category'] ).columns)
 print("numeric_features_train = ", numeric_features_train)
 print("Len = ", len(numeric_features_train))
print("
numeric_features_test = ", numeric_features_test)
 print("Len = ", len(numeric_features_test))
X_train, X_test = X[:len(df_train)], X[len(df_train):]
features_train, features_test = features[:len(df_train)], features[len(df_train):]
df_train, df_test = df.iloc[:len(df_train)], df.iloc[len(df_train):]
y_train = df_train['target'].values
np.random.seed(SEED)
 sample = np.random.permutation(len(X_train))
X_train = X_train[sample]
y_train = y_train[sample]
features_train = features_train[sample] 
<save_model>
print(train.isnull().any())
 print(test.isnull().any() )
np.save("x_train", X_train)
 np.save("x_test", X_test)
 np.save("y_train", y_train)
 np.save("features", features_train)
 np.save("test_features",features_test)
 np.save("word_index.npy",tokenizer.word_index )<load_pretrained>
SP = train.pop('SalePrice' )
X_train = np.load("x_train.npy")
 X_test = np.load("x_test.npy")
 y_train = np.load("y_train.npy")
 features_train = np.load("features.npy")
 features_test = np.load("test_features.npy")
 word_index = np.load("word_index.npy" ).item()<normalization>
def getTheta(r):
X_train, X_test , y_train, y_test = train_test_split(train_X, SP,
test_size = 0.2,
random_state = r)
 theta = np.dot(np.linalg.pinv(X_train), y_train)
 print("Training Error =",rmse(predictTarget(X_train, theta), y_train))
print("Validation Error = ", rmse(predictTarget(X_test, theta), y_test))
return theta
seed_everything()
glove_embeddings = load_glove(word_index)
 paragram_embeddings = load_para(word_index)
 fasttext_embeddings = load_fasttext(word_index)
 embedding_matrix = np.mean([glove_embeddings, paragram_embeddings, fasttext_embeddings], axis=0)
 del glove_embeddings, paragram_embeddings, fasttext_embeddings
gc.collect()
np.shape(embedding_matrix )<train_model>
def predictTarget(x, theta):
return np.dot(x, theta)
 def rmse(pred, actual):
assert(len(pred)== len(actual))
return np.sqrt(((( pred - actual)**2 ).sum())/len(pred))
splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED ).split(X_train, y_train))
splits[:3]<choose_model_class>
test_X = np.hstack(( test.iloc[:,1:],
np.ones(( len(test), 1)) 
))
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs 
<choose_model_class>
prediction = np.zeros(( test_X.shape[0], 1))
rList = [42, 3, 468, 12, 120, 130, 234, 1000, 100, 200, 1200, 5, 45,
49, 300, 320,980, 81, 2100, 1420, 768]
for r in rList:
y1 = predictTarget(test_X, getTheta(r))
print(prediction.shape)
 prediction = np.hstack(( prediction,y1.reshape(y1.shape[0], 1)) )
embedding_dim = 400
embedding_path = '.. /save/embedding_matrix.npy'
use_pretrained_embedding = True
hidden_size = 60
gru_len = hidden_size
Routings = 4
Num_capsule = 5
Dim_capsule = 5
dropout_p = 0.25
rate_drop_dense = 0.28
LR = 0.001
T_epsilon = 1e-7
num_classes = 30
class Embed_Layer(nn.Module):
def __init__(self, embedding_matrix=None, vocab_size=None, embedding_dim=400):
super(Embed_Layer, self ).__init__()
self.encoder = nn.Embedding(vocab_size + 1, embedding_dim)
 if use_pretrained_embedding:
self.encoder.weight.data.copy_(t.from_numpy(embedding_matrix))
def forward(self, x, dropout_p=0.25):
return nn.Dropout(p=dropout_p )(self.encoder(x))
class GRU_Layer(nn.Module):
def __init__(self):
super(GRU_Layer, self ).__init__()
self.gru = nn.GRU(input_size=400,
hidden_size=gru_len,
bidirectional=True)

def init_weights(self):
ih =(param.data for name, param in self.named_parameters() if 'weight_ih' in name)
 hh =(param.data for name, param in self.named_parameters() if 'weight_hh' in name)
 b =(param.data for name, param in self.named_parameters() if 'bias' in name)
 for k in ih:
nn.init.xavier_uniform_(k)
 for k in hh:
nn.init.orthogonal_(k)
 for k in b:
nn.init.constant_(k, 0)
 def forward(self, x):
return self.gru(x)
 class Caps_Layer(nn.Module):
def __init__(self, input_dim_capsule=gru_len * 2, num_capsule=Num_capsule, dim_capsule=Dim_capsule, \
routings=Routings, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Caps_Layer, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = self.squash
else:
self.activation = nn.ReLU(inplace=True)
 if self.share_weights:
self.W = nn.Parameter(
 nn.init.xavier_normal_(t.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))
 else:
self.W = nn.Parameter(
 t.randn(BATCH_SIZE, input_dim_capsule, self.num_capsule * self.dim_capsule))
def forward(self, x):
if self.share_weights:
u_hat_vecs = t.matmul(x, self.W)
 else:
print('add later')
 batch_size = x.size(0)
 input_num_capsule = x.size(1)
 u_hat_vecs = u_hat_vecs.view(( batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = u_hat_vecs.permute(0, 2, 1, 3)
 b = t.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = b.permute(0, 2, 1)
 c = F.softmax(b, dim=2)
 c = c.permute(0, 2, 1)
 b = b.permute(0, 2, 1)
 outputs = self.activation(t.einsum('bij,bijk->bik',(c, u_hat_vecs)))
 if i < self.routings - 1:
b = t.einsum('bik,bijk->bij',(outputs, u_hat_vecs))
return outputs
def squash(self, x, axis=-1):
s_squared_norm =(x ** 2 ).sum(axis, keepdim=True)
 scale = t.sqrt(s_squared_norm + T_epsilon)
 return x / scale
class Capsule_Main(nn.Module):
def __init__(self, embedding_matrix=None, vocab_size=None):
super(Capsule_Main, self ).__init__()
self.embed_layer = Embed_Layer(embedding_matrix, vocab_size)
 self.gru_layer = GRU_Layer()
self.gru_layer.init_weights()
self.caps_layer = Caps_Layer()
self.dense_layer = Dense_Layer()
def forward(self, content):
content1 = self.embed_layer(content)
 content2, _ = self.gru_layer(
 content1)
 content3 = self.caps_layer(content2)
 output = self.dense_layer(content3)
 return output 
<normalization>
prediction = np.sum(prediction, axis = 1)/len(rList )
<categorify><EOS> 
sub = pd.DataFrame()
sub['Id'] = test['Id']
sub['SalePrice'] = prediction
sub.to_csv("prediction.csv", index = False )