Kaggle competition: Quora Question Pairs
%%time
for df in [train, test]:
df['mean_meter_reading_per_building'] = df['mean_meter_reading_per_building'].astype("float16")
 df['median_meter_reading_per_building'] = df['mean_meter_reading_per_building'].astype("float16")
 df['std_meter_reading_per_building'] = df['std_meter_reading_per_building'].astype("float16")
 df['mean_meter_reading_on_year_built'] = df['mean_meter_reading_on_year_built'].astype("float16")
 df['median_meter_reading_on_year_built'] = df['median_meter_reading_on_year_built'].astype("float16")
 df['std_meter_reading_on_year_built'] = df['std_meter_reading_on_year_built'].astype("float16")
 df['mean_meter_reading_per_meter'] = df['mean_meter_reading_per_meter'].astype("float16")
 df['median_meter_reading_per_meter'] = df['median_meter_reading_per_meter'].astype("float16")
 df['std_meter_reading_per_meter'] = df['std_meter_reading_per_meter'].astype("float16")
 df['mean_meter_reading_per_primary_usage'] = df['mean_meter_reading_per_primary_usage'].astype("float16")
 df['median_meter_reading_per_primary_usage'] = df['median_meter_reading_per_primary_usage'].astype("float16")
 df['std_meter_reading_per_primary_usage'] = df['std_meter_reading_per_primary_usage'].astype("float16")
 df['mean_meter_reading_per_site_id'] = df['mean_meter_reading_per_site_id'].astype("float16")
 df['median_meter_reading_per_site_id'] = df['median_meter_reading_per_site_id'].astype("float16")
 df['std_meter_reading_per_site_id'] = df['std_meter_reading_per_site_id'].astype("float16")
 df['number_unique_meter_per_building'] = df['number_unique_meter_per_building'].astype('uint8')
 gc.collect()<categorify>
import pandas as pd
import numpy as np
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding,Activation,Flatten,merge,TimeDistributed,CuDNNGRU,Bidirectional
from keras.layers.merge import concatenate
from keras.models import Model
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.layers import concatenate,subtract,add,maximum,multiply,Layer,Lambda
from keras.backend import backend as K
from tqdm import tqdm
train.drop('timestamp',axis=1,inplace=True)
 test.drop('timestamp',axis=1,inplace=True)
 le = LabelEncoder()
train['meter']= le.fit_transform(train['meter'] ).astype("uint8")
 test['meter']= le.fit_transform(test['meter'] ).astype("uint8")
 train['primary_use']= le.fit_transform(train['primary_use'] ).astype("uint8")
 test['primary_use']= le.fit_transform(test['primary_use'] ).astype("uint8")
 print(train.shape, test.shape )<prepare_x_and_y>
MAX_SEQUENCE_LENGTH = 40
MAX_NB_WORDS = 4000000
nrows = 10000000
EMBEDDING_DIM = 300
path = '.. /input/quora-question-pairs/'
train = pd.read_csv(path+"train.csv",nrows=nrows ).astype(str)
 test = pd.read_csv(path+"test.csv",nrows=nrows ).astype(str )
%%time
y = train['meter_reading']
train.drop('meter_reading',axis=1,inplace=True)
 categorical_cols = ['building_id','Month','meter','Hour','primary_use','DayOfWeek','DayOfMonth']<split>
corpus = []
feats = ['question1','question2']
for f in feats:
train[f] = train[f].astype(str)
 test[f] = test[f].astype(str)
 corpus+=train[f].values.tolist()
tokenizer = Tokenizer(num_words=MAX_NB_WORDS)
 tokenizer.fit_on_texts(corpus)
 X_q1 = tokenizer.texts_to_sequences(train['question1'])
 X_q2 = tokenizer.texts_to_sequences(train['question2'])
 X_test_q1 = tokenizer.texts_to_sequences(test['question1'])
 X_test_q2 = tokenizer.texts_to_sequences(test['question2'])
 X_q1 = pad_sequences(X_q1, maxlen=MAX_SEQUENCE_LENGTH)
 X_q2 = pad_sequences(X_q2, maxlen=MAX_SEQUENCE_LENGTH)
 X_test_q1 = pad_sequences(X_test_q1, maxlen=MAX_SEQUENCE_LENGTH)
 X_test_q2 = pad_sequences(X_test_q2, maxlen=MAX_SEQUENCE_LENGTH)
 y = train['is_duplicate'].values
word_index = tokenizer.word_index
print('Found %s unique tokens' % len(word_index))
nb_words = min(MAX_NB_WORDS, len(word_index)) +1
x_train,x_test,y_train,y_test = train_test_split(train,y,test_size=0.2,random_state=42, stratify=meter_cut)
 print(x_train.shape)
 print(y_train.shape)
 print(x_test.shape)
 print(y_test.shape )<import_modules>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
glove_path = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
embedding_matrix,unknown_words = build_matrix(word_index,glove_path )
from sklearn.ensemble import RandomForestRegressor as RF
import lightgbm as lgb<create_dataframe>
from sklearn.metrics import log_loss
lgb_train = lgb.Dataset(x_train, y_train,categorical_feature=categorical_cols)
 lgb_test = lgb.Dataset(x_test, y_test,categorical_feature=categorical_cols)
<init_hyperparams>
X_train_q1,X_val_q1,X_train_q2,X_val_q2,y_train,y_val = train_test_split(X_q1,X_q2,y,train_size=0.8,random_state=1024)
 print(X_train_q1.shape,X_val_q1.shape)
 X_train = [X_train_q1,X_train_q2]
X_val = [X_val_q1,X_val_q2]
X_test = [X_test_q1,X_test_q2]
del x_train, y_train
params = {'feature_fraction': 0.75,
'bagging_fraction': 0.75,
'objective': 'regression_l2',
'max_depth': -1,
'learning_rate': 0.01,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'rmse',
"verbosity": -1,
'reg_alpha': 0.5,
'reg_lambda': 0.5,
'random_state': 47
}
reg = lgb.train(params, lgb_train, num_boost_round=3000, valid_sets=[lgb_train, lgb_test], early_stopping_rounds=100, verbose_eval = 100 )<drop_column>
def get_model_siamese(nb_words=50000,MAX_SEQUENCE_LENGTH=200,EMBEDDING_DIM=200,act='relu',embedding_matrix=None):
input_q1 = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
 input_q2 = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
 if embedding_matrix is not None:
weights=[embedding_matrix],
embedding_layer = Embedding(nb_words,
EMBEDDING_DIM,
input_length=MAX_SEQUENCE_LENGTH,
weights=[embedding_matrix],
trainable=False)
 else:
embedding_layer = Embedding(nb_words,
EMBEDDING_DIM,
input_length=MAX_SEQUENCE_LENGTH,
trainable=True)
 embedded_sequences_q1 = embedding_layer(input_q1)
 embedded_sequences_q2 = embedding_layer(input_q2)
 bilstm_layer = Bidirectional(CuDNNGRU(64,return_sequences=False))
x1 = bilstm_layer(embedded_sequences_q1)
 x2 = bilstm_layer(embedded_sequences_q2)
 diff = subtract([x1,x2])
 summation = add([x1,x2])
 x = concatenate([x1,x2,diff,summation],1)
 x = Dense(64,activation=act )(x)
 preds = Dense(1,activation='sigmoid' )(x)
 model = Model(inputs=[input_q1,input_q2],outputs=preds)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 print(model.summary())
 return model
del train<predict_on_test>
early_stopping =EarlyStopping(monitor='val_loss', patience=5)
 bst_model_path = 'best_model.h5'
model_checkpoint = ModelCheckpoint(bst_model_path, save_best_only=True, save_weights_only=True)
 model = get_model_siamese(nb_words=nb_words,MAX_SEQUENCE_LENGTH=MAX_SEQUENCE_LENGTH,EMBEDDING_DIM=EMBEDDING_DIM,act='relu',embedding_matrix=embedding_matrix)
 model.fit(X_train,y_train,epochs=250,validation_data=(X_val, y_val),batch_size=128,shuffle=True,verbose=1,callbacks=[early_stopping, model_checkpoint])
 model.load_weights(bst_model_path)
<compute_test_metric><EOS> 
y_pred_test = model.predict(X_test)
 submission = pd.DataFrame()
submission['test_id'] = test['test_id']
submission['is_duplicate'] = y_pred_test
submission.to_csv('submission_siamese.csv',index=False)