Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<concatenate>
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from keras.models import Model
from keras.layers import Dense, Input
from keras.layers.merge import concatenate
from keras.optimizers import SGD
embedding_matrix = np.concatenate([fastText_embedding_index, glove_embedding_index,glove_twitter], axis=-1)
 embedding_matrix.shape
del glove_embedding_index
del fastText_embedding_index
del glove_twitter
gc.collect()<define_variables>
def organize_data(X):
max_non_null_terms = 0
num_rows = X.shape[0]
for i in range(0,num_rows):
temp = np.array(sorted(X[i,:],reverse=True))
X[i,:] = temp
non_null_terms = np.sum(X[i,:] > 0)
 if non_null_terms > max_non_null_terms:
max_non_null_terms = non_null_terms
X = X[:,0:max_non_null_terms]
return X
word_vec_size=300<categorify>
def normalize_data(X):
num_rows = X.shape[0]
num_cols = X.shape[1]
X_normalized = np.zeros(( num_rows,num_cols), dtype = np.float32)
 for i in range(0,num_rows):
for j in range(0,num_cols):
if X[i,j] > 0:
X_normalized[i,j] = np.log10(X[i,j])
 return X_normalized
def get_embedding_matrix(word_index, embedding_index, word_vec_size):
toxinizer_vocab = len(word_index)+1
embedding_matrix = np.zeros(( toxinizer_vocab, word_vec_size))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return(embedding_matrix )<set_options>
train_data = pd.read_csv(".. /input/train.csv")
 train_val = train_data.values
Y = train_val[:,1].astype(np.float32)
 X = train_val[:,2:].astype(np.float32)
 test_data = pd.read_csv(".. /input/test.csv")
 test_val = test_data.values
X_test = test_val[:,1:].astype(np.float32 )
def seed_everything(seed=1234):
random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 torch.backends.cudnn.deterministic = True<normalization>
X_all = np.vstack(( X, X_test))
X_all = organize_data(X_all)
 num_features = X_all.shape[1]
X_all = normalize_data(X_all)
 Y = Y.reshape(Y.shape[0],1)
 Y_normalized = normalize_data(Y )
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x<groupby>
num_new_features = 5
X_add = np.zeros(( X_all.shape[0],num_features + num_new_features), dtype = np.float32)
 for i in range(0,X_all.shape[0]):
X_add[i,0] = np.mean(X_all[i,X_all[i,:]!=0])
 X_add[i,1] = np.min(X_all[i,X_all[i,:]!=0])
 X_add[i,2] = np.max(X_all[i,:])
 X_add[i,3] = np.sum(X_all[i,:] > 0)
 X_add[i,4] = np.std(X_all[i,:])
 X_add[:,5:] = X_all
num_features = num_features + num_new_features
def weighted_avg(tensor,dim):
weight_array = np.arange(1,tensor.shape[dim] + 1)
 weight_array = weight_array / np.sum(weight_array)
 weight_array = torch.from_numpy(weight_array ).float().cuda()
for a in range(len(tensor.shape)) :
if a != dim:
weight_array = weight_array.unsqueeze(a)
 weighted_tensor = tensor * weight_array
return torch.sum(weighted_tensor,dim)
<choose_model_class>
X_train_final = X_add[0:X.shape[0],:]
X_test_final = X_add[X.shape[0]:,:]
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, dropout_rate,
lstm_units, dense_hidden_units):
super(NeuralNet, self ).__init__()
vocab_size = embedding_matrix.shape[0]
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(vocab_size, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix,
dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(dropout_rate)
 self.lstm1 = nn.LSTM(embed_size,lstm_units,bidirectional=True,batch_first=True,num_layers=2)
 self.linear1 = nn.Linear(dense_hidden_units, dense_hidden_units)
 self.linear2 = nn.Linear(dense_hidden_units, dense_hidden_units)
 self.dropout = nn.Dropout (.2)
 self.linear_out = nn.Linear(dense_hidden_units, 1)
 self.bn = nn.BatchNorm1d(dense_hidden_units)
 def forward(self, x, lengths=None):
h_embedding = self.embedding(x.long())
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm2, _ = self.lstm1(h_embedding)
 avg_pool_1 = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool_1), 1)
 h_conc_linear1 = F.relu(self.bn(self.linear1(h_conc)))
 h_conc_linear2 = self.dropout(F.relu(self.bn(self.linear2(h_conc))))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 return result<compute_test_metric>
for cycles in [1,5,10]:
for neurons in [10,50,100,150]:
visible = Input(shape=(num_features,))
hid1 = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hid2 = Dense(neurons, kernel_initializer='normal', activation = "relu")
 hidpar2 = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hid3 = Dense(neurons, kernel_initializer='normal', activation = "relu")
 hidpar3 = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hid1_ = Dense(neurons, kernel_initializer='normal', activation = "relu")
 hid2_ = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hidpar2_ = Dense(neurons, kernel_initializer='normal', activation = "relu")
 hid3_ = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hidpar3_ = Dense(neurons, kernel_initializer='normal', activation = "relu")
 feedback_hid = Dense(num_features, kernel_initializer='normal', activation = "relu")
 x = visible
L = []
LP = []
for i in range(0,cycles):
L.append(hid1(x))
L.append(hid2(L[0]))
L.append(hidpar2(L[1]))
L.append(hid3(L[2]))
L.append(hidpar3(L[3]))
L.append(concatenate([L[3],L[4]]))
LP.append(hid1_(x))
LP.append(hid2_(LP[0]))
LP.append(hidpar2_(LP[1]))
LP.append(hid3_(LP[2]))
LP.append(hidpar3_(LP[3]))
LP.append(concatenate([LP[3],LP[4]]))
final_merge = concatenate([L[-1],LP[-1]])
 x = feedback_hid(final_merge)
 master = Dense(neurons, kernel_initializer='normal', activation='tanh' )(x)
 output = Dense(1, kernel_initializer='normal', activation='softplus' )(master)
 model = Model(inputs=visible, outputs=output)
 sgd = SGD(lr=0.01, momentum=0.1, decay=0.0, nesterov=False)
 model.compile(loss='mean_squared_error', optimizer=sgd)
 history = model.fit(X_train_final, Y_normalized,validation_split=0.33, epochs=150, batch_size=100, verbose=0)
 print('Cycles =',cycles)
 print('Neurons =', neurons)
 print('
Loss of predictions in train dataset:')
 predictions = model.predict(X_train_final)
 predictions = 10**predictions
print(np.sqrt(1/Y.shape[0])*np.linalg.norm(( np.log(predictions+1)-np.log(Y+1)) ,2))
plt.plot(history.history['loss'][2:])
 plt.plot(history.history['val_loss'][2:])
 plt.title('model loss')
 plt.ylabel('loss')
 plt.xlabel('epoch')
 plt.legend(['train', 'test'], loc='upper left')
 plt.show()
def custom_loss(data, targets, weights):

 bce_loss_1 = nn.BCEWithLogitsLoss(weight=weights.float() )(data[:,:1].float() ,targets[:,:1].float())
 return bce_loss_1<compute_test_metric>
cycles = 5
neurons = 150
visible = Input(shape=(num_features,))
hid1 = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hid2 = Dense(neurons, kernel_initializer='normal', activation = "relu")
 hidpar2 = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hid3 = Dense(neurons, kernel_initializer='normal', activation = "relu")
 hidpar3 = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hid1_ = Dense(neurons, kernel_initializer='normal', activation = "relu")
 hid2_ = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hidpar2_ = Dense(neurons, kernel_initializer='normal', activation = "relu")
 hid3_ = Dense(neurons, kernel_initializer='normal', activation = "tanh")
 hidpar3_ = Dense(neurons, kernel_initializer='normal', activation = "relu")
 feedback_hid = Dense(num_features, kernel_initializer='normal', activation = "relu")
 mast = Dense(neurons, kernel_initializer='normal', activation='tanh')
 out = Dense(1, kernel_initializer='normal', activation='softplus')
 x = visible
L = []
LP = []
for i in range(0,cycles):
L.append(hid1(x))
L.append(hid2(L[0]))
L.append(hidpar2(L[1]))
L.append(hid3(L[2]))
L.append(hidpar3(L[3]))
L.append(concatenate([L[3],L[4]]))
LP.append(hid1_(x))
LP.append(hid2_(LP[0]))
LP.append(hidpar2_(LP[1]))
LP.append(hid3_(LP[2]))
LP.append(hidpar3_(LP[3]))
LP.append(concatenate([LP[3],LP[4]]))
final_merge = concatenate([L[-1],LP[-1]])
 x = feedback_hid(final_merge)
 master = mast(x)
 output = out(master)
 model = Model(inputs=visible, outputs=output)
 sgd = SGD(lr=0.01, momentum=0.1, decay=0.0, nesterov=False)
 model.compile(loss='mean_squared_error', optimizer=sgd )
<train_model><EOS> 
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission["target"] = predictions
submission.to_csv('submission.csv', index=False)
 print(submission.head() )