Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<data_type_conversions>
import numpy as np
import pandas as pd
import seaborn as sns
for i,(train_idx, valid_idx)in enumerate(splits):
x_train = np.array(x_train)
 y_train = np.array(y_train)
 features = np.array(features)
 x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
kfold_X_features = features[train_idx.astype(int)]
kfold_X_valid_features = features[valid_idx.astype(int)]
x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')
 step_size = 300
base_lr, max_lr = 0.001, 0.003
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=max_lr)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train = MyDataset(train)
 valid = MyDataset(valid)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for i,(x_batch, y_batch, index)in enumerate(train_loader):
f = kfold_X_features[index]
y_pred = model([x_batch,f])
 if scheduler:
scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(df_test)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch, index)in enumerate(valid_loader):
f = kfold_X_valid_features[index]
y_pred = model([x_batch,f] ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
avg_losses_f.append(avg_loss)
 avg_val_losses_f.append(avg_val_loss)
 for i,(x_batch,)in enumerate(test_loader):
f = test_features[i * batch_size:(i+1)* batch_size]
y_pred = model([x_batch,f] ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 print('All \t loss={:.4f} \t val_loss={:.4f} \t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))
<compute_test_metric>
train_raw_data=pd.read_csv('.. /input/titanic/train.csv')
 test_raw_data=pd.read_csv('.. /input/titanic/test.csv' )
def bestThresshold(y_train,train_preds):
tmp = [0,0,0]
delta = 0
for tmp[0] in tqdm(np.arange(0.1, 0.501, 0.01)) :
tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))
return delta
delta = bestThresshold(y_train,train_preds )<save_to_csv>
trainrow=train_raw_data.shape[0]
testrow=test_raw_data.shape[0]
y_train=train_raw_data['Survived'].copy()
train_raw_data=train_raw_data.drop(['Survived'],1 )
submission = df_test[['qid']].copy()
submission['prediction'] =(test_preds > delta ).astype(int)
 submission.to_csv('submission.csv', index=False )<import_modules>
combine=pd.concat([train_raw_data,test_raw_data])
 combine.head()
from sklearn.model_selection import GridSearchCV,StratifiedKFold
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import cross_val_score,train_test_split
from scipy import stats
from sklearn import metrics
from keras.models import Sequential
from keras.layers import Dense
from keras.preprocessing.text import Tokenizer
import string,re
from collections import Counter
import nltk
from nltk.corpus import stopwords
import spacy
from spacy.lang.en.stop_words import STOP_WORDS
from spacy.lang.en import English<import_modules>
combine.isnull().sum()
import time
from tqdm import tqdm
import math
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D,CuDNNLSTM
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
from sklearn import model_selection as ms
from sklearn.model_selection import KFold,StratifiedKFold
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.svm import LinearSVC
from keras.layers import Input, Dense, Embedding, concatenate
from keras.layers import CuDNNGRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D
from keras.preprocessing import text, sequence
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.layers import *
from keras.callbacks import *<load_from_csv>
combine['Embarked']=combine['Embarked'].fillna(combine['Embarked'].value_counts().index[0] )
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 print("train data shape --",df_train.shape)
 print("test data shape --",df_test.shape )<feature_engineering>
combine['Cabin']=combine['Cabin'].fillna('U')
 combine['Cabin'].value_counts()
combine['Cabin']=combine['Cabin'].astype(str ).str[0]
combine.head()
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('.',' fullstop '))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('?',' endofquestion '))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace(',',' comma '))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('!',' exclamationmark '))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('-',' hyphen '))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('/',' backslash '))<feature_engineering>
combine.loc[combine['Fare'].isnull() ]
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('fullstop','.'))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('endofquestion','?'))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('comma',','))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('exclamationmark','!'))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('hyphen','-'))
df_train["question_text"] = df_train["question_text"].apply(lambda x: x.replace('backslash','/'))<feature_engineering>
combine['Fare']=combine['Fare'].fillna(combine.loc[(combine['Pclass']==3)&(combine['Sex']=="male")&(combine['Age']<65)&(combine['Age']>55)].dropna() ['Fare'].mean() )
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('.',' fullstop '))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('?',' endofquestion '))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace(',',' comma '))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('!',' exclamationmark '))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('-',' hyphen '))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('/',' backslash '))<feature_engineering>
passengerids=test_raw_data['PassengerId']
combine=combine.drop(['PassengerId','Ticket'],1 )
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('fullstop','.'))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('endofquestion','?'))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('comma',','))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('exclamationmark','!'))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('hyphen','-'))
df_test["question_text"] = df_test["question_text"].apply(lambda x: x.replace('backslash','/'))<concatenate>
combine['familysize']=combine['SibSp']+combine['Parch']+1
combine.head()
df_combined = pd.concat([df_train,df_test],axis=0)
 print("combined shape ",df_combined.shape )<define_variables>
combine['Title'] = combine.Name.str.extract('([A-Za-z]+)\.', expand=False)
 combine.head()
embed_size = 300
max_features = 60000
maxlen = 60
total_X = df_combined["question_text"].values<feature_engineering>
combine['Title'].value_counts()
tokenizer = Tokenizer(num_words=max_features,filters='"
',)
 tokenizer.fit_on_texts(list(total_X))<count_values>
combine=combine.drop(['Name'],1)
 combine.head()
WORDS = tokenizer.word_counts
print(len(WORDS))<prepare_x_and_y>
combine=combine.drop(['SibSp','Parch'],1)
 combine.head()
train_X = df_train["question_text"].values
test_X = df_test["question_text"].values<string_transform>
combine['Sex']=combine['Sex'].map({'male':0,'female':1})
 combine.head()
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X )<prepare_x_and_y>
for i in range(0,2):
for j in range(0,3):
print(i,j+1)
 temp_dataset=combine[(combine['Sex']==i)&(combine['Pclass']==j+1)]['Age'].dropna()
print(temp_dataset)
 combine.loc[(combine.Age.isnull())&(combine.Sex==i)&(combine.Pclass==j+1),'Age']=int(temp_dataset.median() )
train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = df_train['target'].values<categorify>
combine.isnull().sum()
def get_embeddings(embedtype):
if embedtype is "glove":
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
elif embedtype is "fastext":
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
elif embedtype is "paragram":
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE,encoding="latin")if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
vocab_size = all_embs.shape[0]
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
print("embed size-->",embed_size)
 print("total words in embeddings-->",vocab_size)
 print("total words in data-->",len(word_index))
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
count_common_words =0
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
count_common_words = count_common_words+1
print(count_common_words," common words fount in ",embedtype)
 print("{0}% common words found in {1}".format(( count_common_words*100/len(word_index)) ,embedtype))
return embedding_matrix 
<choose_model_class>
combine_checkpoint=combine.copy()
combine.head()
embedding_glove = get_embeddings(embedtype="glove" )<choose_model_class>
combine['Age_Band']=pd.cut(combine['Age'],5)
 combine['Age_Band'].unique()
embedding_paragram = get_embeddings(embedtype="paragram" )<train_model>
combine.loc[(combine['Age']<=16.136),'Age']=1
combine.loc[(combine['Age']>16.136)&(combine['Age']<=32.102),'Age']=2
combine.loc[(combine['Age']>32.102)&(combine['Age']<=48.068),'Age']=3
combine.loc[(combine['Age']>48.068)&(combine['Age']<=64.034),'Age']=4
combine.loc[(combine['Age']>64.034)&(combine['Age']<=80.) ,'Age']=5
combine['Age'].unique()
mean_gl_par_embedding = np.mean([embedding_glove,embedding_paragram],axis=0)
 print("mean glove paragram embedding shape--> ",mean_gl_par_embedding.shape )<set_options>
combine=combine.drop(['Age_Band'],1 )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
combine['Fare_Band']=pd.cut(combine['Fare'],3)
 combine['Fare_Band'].unique()
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr() )<choose_model_class>
combine.loc[(combine['Fare']<=170.776),'Fare']=1
combine.loc[(combine['Fare']>170.776)&(combine['Fare']<=314.553),'Fare']=2
combine.loc[(combine['Fare']>314.553)&(combine['Fare']<=513),'Fare']=3
combine=combine.drop(['Fare_Band'],1 )
def build_model() :
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[mean_gl_par_embedding],trainable=False )(inp)
 x = SpatialDropout1D(rate=0.1 )(x)
 x1 = Bidirectional(CuDNNGRU(200, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 atten_1 = Attention(maxlen )(x1)
 atten_2 = Attention(maxlen )(x2)
 x = concatenate([atten_1,atten_2])
 x = Dense(128, activation="relu" )(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model<compute_test_metric>
combine['Fare'].value_counts()
def f1_smart(y_true, y_pred):
args = np.argsort(y_pred)
 tp = y_true.sum()
fs =(tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)
 res_idx = np.argmax(fs)
 return 2 * fs[res_idx],(y_pred[args[res_idx]] + y_pred[args[res_idx + 1]])/ 2<split>
combine=pd.get_dummies(columns=['Pclass','Sex','Cabin','Embarked','Title','Age','Fare'],data=combine)
 combine.head()
kfold = StratifiedKFold(n_splits=5, random_state=1990, shuffle=True)
 bestscore = []
y_test = np.zeros(( test_X.shape[0],))
filepath="weights_best_mean.h5"
for i,(train_index, valid_index)in enumerate(kfold.split(train_X, train_y)) :
X_train, X_val, Y_train, Y_val = train_X[train_index], train_X[valid_index], train_y[train_index], train_y[valid_index]
checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.6, patience=1, min_lr=0.0001, verbose=2)
 earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')
 callbacks = [clr,]
model = build_model()
if i == 0:print(model.summary())
 model.fit(X_train, Y_train, batch_size=512, epochs=2, validation_data=(X_val, Y_val), verbose=1, callbacks=[checkpoint,clr,])
 model.load_weights(filepath)
 y_pred = model.predict([X_val], batch_size=1024, verbose=1)
 y_test += np.squeeze(model.predict([test_X], batch_size=1024, verbose=2)) /5
f1, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1, threshold))
bestscore.append(threshold )<compute_test_metric>
x_train=combine.iloc[:trainrow]
x_test=combine.iloc[trainrow:]
print("mean threshold--> ",np.mean(bestscore))<save_to_csv>
from sklearn.preprocessing import StandardScaler
print(y_test.shape)
 pred_test_y =(y_test>np.mean(bestscore)).astype(int)
 out_df = pd.DataFrame({"qid":df_test["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<import_modules>
scaler=StandardScaler()
scaler.fit(x_train)
 x_scaled_train=scaler.transform(x_train)
 x_scaled_train
tqdm.pandas(desc='Progress')
<set_options>
x_scaled_test=scaler.transform(x_test)
 x_scaled_test
def seed_everything(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
SEED=12345<define_variables>
reg=LogisticRegression()
reg.fit(x_scaled_train,y_train)
 print(reg.score(x_scaled_train,y_train))
y_pred=reg.predict(x_scaled_test)
 y_pred
embed_size = 300
max_features = 120000
maxlen = 80
batch_size = 256
n_epochs = 5
n_splits = 5 
<train_model>
xgb=XGBClassifier()
xgb.fit(x_scaled_train,y_train,early_stopping_rounds=5,
eval_set=[(x_scaled_train, y_train)],
verbose=False)
 print(xgb.score(x_scaled_train,y_train))
y_pred=xgb.predict(x_scaled_test )
token = Tokenizer()
token.fit_on_texts(["Let us learn on a example"])
 print(token.texts_to_sequences(["Let us learn on a example"]))
print(token.texts_to_sequences(["Let us hopefully learn on a example"]))<drop_column>
rfc=RandomForestClassifier(random_state=4,n_estimators=500,warm_start=True,max_depth=6,min_samples_leaf=2,max_features='sqrt')
 rfc.fit(x_scaled_train,y_train)
 print(rfc.score(x_scaled_train,y_train))
y_pred=rfc.predict(x_scaled_test )
del token<load_from_csv>
submission = pd.DataFrame({
"PassengerId": passengerids,
"Survived": y_pred
})
 submission
df_train = pd.read_csv(".. /input/quora-insincere-questions-classification/train.csv")
 df_test = pd.read_csv(".. /input/quora-insincere-questions-classification/test.csv")
 df = pd.concat([df_train ,df_test],sort=True )<feature_engineering>
submission.to_csv('submission1.csv', index=False )