Kaggle competition: Titanic - Machine Learning from Disaster
class CBOW(nn.Module):
def __init__(self, num_embeddings, embedding_dim):
super().__init__()
self.emb = nn.Embedding(num_embeddings, embedding_dim, padding_idx=0)
 self.lin = nn.Linear(embedding_dim, num_embeddings, bias=False)
 def forward(self, input):
e = self.emb(input)
 u = e.sum(dim=1)
 v = self.lin(u)
 return v<categorify>
df = pd.read_csv('/kaggle/input/titanic/train.csv')
 df2 = pd.read_csv('/kaggle/input/titanic/test.csv')
 pd.options.display.max_columns = None
df.head()
def load_preprocessed_dataset(prefix):
token_vocab = Vocabulary()
token_vocab.load(f'{prefix}.vocab')
 data = []
for part in ['train', 'valid', 'test']:
with np.load(f'{prefix}.{part}.npz')as set_data:
idata, target = set_data['idata'], set_data['target']
data.append(( idata, target))
print(f'Number of samples({part}): {len(target)}')
 print("Using precomputed vocabulary and data files")
 print(f'Vocabulary size: {len(token_vocab)}')
 return token_vocab, data<train_model>
df3 = pd.read_csv('/kaggle/input/titanic/gender_submission.csv')
 df3.head()
def train(model, criterion, optimizer, idata, target, batch_size, device, log=False):
model.train()
total_loss = 0
ncorrect = 0
ntokens = 0
niterations = 0
for X, y in batch_generator(idata, target, batch_size, shuffle=True):
X = torch.tensor(X, dtype=torch.long, device=device)
 y = torch.tensor(y, dtype=torch.long, device=device)
 model.zero_grad()
output = model(X)
 loss = criterion(output, y)
 loss.backward()
optimizer.step()
total_loss += loss.item()
ncorrect +=(torch.max(output, 1)[1] == y ).sum().item()
ntokens += y.numel()
niterations += 1
if niterations == 200 or niterations == 500 or niterations % 1000 == 0:
print(f'Train: wpb={ntokens//niterations}, num_updates={niterations}, accuracy={100*ncorrect/ntokens:.1f}, loss={total_loss/ntokens:.2f}')
 total_loss = total_loss / ntokens
accuracy = 100 * ncorrect / ntokens
if log:
print(f'Train: wpb={ntokens//niterations}, num_updates={niterations}, accuracy={accuracy:.1f}, loss={total_loss:.2f}')
 return accuracy, total_loss<train_model>
percent_missing = df.isnull().sum() * 100 / len(df)
 missing_values = pd.DataFrame({'percent_missing': percent_missing})
 missing_values.sort_values(by ='percent_missing' , ascending=False )
def validate(model, criterion, idata, target, batch_size, device):
model.eval()
total_loss = 0
ncorrect = 0
ntokens = 0
niterations = 0
y_pred = []
with torch.no_grad() :
for X, y in batch_generator(idata, target, batch_size, shuffle=False):
X = torch.tensor(X, dtype=torch.long, device=device)
 output = model(X)
 if target is not None:
y = torch.tensor(y, dtype=torch.long, device=device)
 loss = criterion(output, y)
 total_loss += loss.item()
ncorrect +=(torch.max(output, 1)[1] == y ).sum().item()
ntokens += y.numel()
niterations += 1
else:
pred = torch.max(output, 1)[1].detach().to('cpu' ).numpy()
y_pred.append(pred)
 if target is not None:
total_loss = total_loss / ntokens
accuracy = 100 * ncorrect / ntokens
return accuracy, total_loss
else:
return np.concatenate(y_pred )<set_options>
percent_missing = df2.isnull().sum() * 100 / len(df2)
 missing_values = pd.DataFrame({'percent_missing': percent_missing})
 missing_values.sort_values(by ='percent_missing' , ascending=False )
if torch.cuda.is_available() :
device = torch.device('cuda')
 else:
device = torch.device('cpu')
 print("WARNING: Training without GPU can be very slow!" )<create_dataframe>
df['Survived'].value_counts()
vocab, data = load_preprocessed_dataset(params.preprocessed )<choose_model_class>
X = df.drop(['Cabin', 'Name', 'PassengerId', 'Survived', 'Ticket'], axis = 1)
 Y = df['Survived']
Test_Data = df2.drop(['Cabin', 'Name', 'PassengerId', 'Ticket'], axis = 1 )
model = CBOW(len(vocab), params.embedding_dim ).to(device )<load_from_csv>
X = pd.get_dummies(X, prefix_sep='_')
 Test_Data = pd.get_dummies(Test_Data, prefix_sep='_')
 X.head()
valid_x_df = pd.read_csv(f'{COMPETITION_ROOT}/x_valid.csv')
 tokens = valid_x_df.columns[1:]
valid_x = valid_x_df[tokens].apply(vocab.get_index ).to_numpy(dtype='int32')
 valid_y_df = pd.read_csv(f'{COMPETITION_ROOT}/y_valid.csv')
 valid_y = valid_y_df['token'].apply(vocab.get_index ).to_numpy(dtype='int32' )<choose_model_class>
Test_Data['Embarked_0'] = 0
Test_Data = Test_Data[['Pclass', 'Age','SibSp','Parch','Fare','Sex_female', 'Sex_male', 'Embarked_C', 'Embarked_Q', 'Embarked_S']]
Test_Data.head()
optimizer = torch.optim.Adam(model.parameters())
 criterion = nn.CrossEntropyLoss(reduction='sum')
 train_accuracy = []
wiki_accuracy = []
valid_accuracy = []
for epoch in range(params.epochs):
acc, loss = train(model, criterion, optimizer, data[0][0], data[0][1], params.batch_size, device, log=True)
 train_accuracy.append(acc)
 print(f'| epoch {epoch:03d} | train accuracy={acc:.1f}%, train loss={loss:.2f}')
 acc, loss = validate(model, criterion, data[1][0], data[1][1], params.batch_size, device)
 wiki_accuracy.append(acc)
 print(f'| epoch {epoch:03d} | valid accuracy={acc:.1f}%, valid loss={loss:.2f}(wikipedia)')
 acc, loss = validate(model, criterion, valid_x, valid_y, params.batch_size, device)
 valid_accuracy.append(acc)
 print(f'| epoch {epoch:03d} | valid accuracy={acc:.1f}%, valid loss={loss:.2f}(El Periódico)')
 torch.save(model.state_dict() , params.modelname )<load_from_csv>
Y = LabelEncoder().fit_transform(Y)
 Y
valid_x_df = pd.read_csv(f'{COMPETITION_ROOT}/x_test.csv')
 test_x = valid_x_df[tokens].apply(vocab.get_index ).to_numpy(dtype='int32')
 y_pred = validate(model, None, test_x, None, params.batch_size, device)
 y_token = [vocab.idx2token[index] for index in y_pred]<save_to_csv>
X2 = StandardScaler().fit_transform(X)
 Test_Data = StandardScaler().fit_transform(Test_Data)
 X_Train, X_Test, Y_Train, Y_Test = train_test_split(X2, Y, test_size = 0.20, random_state = 101 )
submission = pd.DataFrame({'id':valid_x_df['id'], 'token': y_token}, columns=['id', 'token'])
 print(submission.head())
 submission.to_csv('submission.csv', index=False )<set_options>
start = time.process_time()
trainedmodel = LogisticRegression().fit(X_Train,Y_Train)
 print(time.process_time() - start)
 predictions =trainedmodel.predict(X_Test)
 print(confusion_matrix(Y_Test,predictions))
print(classification_report(Y_Test,predictions))
%matplotlib inline<load_from_csv>
start = time.process_time()
trainedsvm = svm.LinearSVC().fit(X_Train, Y_Train)
 print(time.process_time() - start)
 predictionsvm = trainedsvm.predict(X_Test)
 print(confusion_matrix(Y_Test,predictionsvm))
print(classification_report(Y_Test,predictionsvm))
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
start = time.process_time()
trainedtree = tree.DecisionTreeClassifier().fit(X_Train, Y_Train)
 print(time.process_time() - start)
 predictionstree = trainedtree.predict(X_Test)
 print(confusion_matrix(Y_Test,predictionstree))
print(classification_report(Y_Test,predictionstree))
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<count_missing_values>
start = time.process_time()
trainedforest = RandomForestClassifier(n_estimators=700 ).fit(X_Train,Y_Train)
 print(time.process_time() - start)
 predictionforest = trainedforest.predict(X_Test)
 print(confusion_matrix(Y_Test,predictionforest))
print(classification_report(Y_Test,predictionforest))
train.isnull().sum()<count_missing_values>
pca = PCA(n_components=2,svd_solver='full')
 X_pca = pca.fit_transform(X)
 X_reduced, X_test_reduced, Y_reduced, Y_test_reduced = train_test_split(X_pca, Y, test_size = 0.30, random_state = 101)
 start = time.process_time()
trainedforest = RandomForestClassifier(n_estimators=700 ).fit(X_reduced,Y_reduced)
 print(time.process_time() - start)
 predictionforest = trainedforest.predict(X_test_reduced)
 print(confusion_matrix(Y_test_reduced,predictionforest))
print(classification_report(Y_test_reduced,predictionforest))
test.isnull().sum()<count_missing_values>
model1 = svm.LinearSVC()
model2 = tree.DecisionTreeClassifier()
model3 = RandomForestClassifier()
model = VotingClassifier(estimators=[('svm', model1),('dt', model2),('rf', model3)], voting='hard')
 model.fit(X_Train,Y_Train)
 model.score(X_Test,Y_Test )
test.isnull().sum()<count_values>
model1 = LogisticRegression()
model2 = tree.DecisionTreeClassifier()
model3 = RandomForestClassifier()
model1.fit(X_Train,Y_Train)
 model2.fit(X_Train,Y_Train)
 model3.fit(X_Train,Y_Train)
 pred1 = model1.predict_proba(X_Test)[:,1]
pred2 = model2.predict_proba(X_Test)[:,1]
pred3 = model3.predict_proba(X_Test)[:,1]
finalpred =(pred1+pred2+pred3)/3
preds = np.where(finalpred > 0.5, 1, 0)
 print(confusion_matrix(Y_Test, preds))
print(classification_report(Y_Test, preds))
train.Outcome.value_counts()<prepare_x_and_y>
model1 = LogisticRegression()
model2 = tree.DecisionTreeClassifier()
model3 = RandomForestClassifier()
model1.fit(X_Train,Y_Train)
 model2.fit(X_Train,Y_Train)
 model3.fit(X_Train,Y_Train)
 pred1 = model1.predict_proba(X_Test)[:,1]
pred2 = model2.predict_proba(X_Test)[:,1]
pred3 = model3.predict_proba(X_Test)[:,1]
finalpred=(pred1*0.2+pred2*0.4+pred3*0.4)
 preds = np.where(finalpred > 0.5, 1, 0)
 print(confusion_matrix(Y_Test, preds))
print(classification_report(Y_Test, preds))
X = train.drop(['Outcome'], axis = 1)
 y = train.Outcome<train_model>
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from xgboost import XGBClassifier
from vecstack import stacking
from sklearn.metrics import accuracy_score
clf = GradientBoostingClassifier(n_estimators = 105)
 clf.fit(X,y )<predict_on_test>
random_search = {'criterion': ['entropy', 'gini'],
'max_depth': list(np.linspace(10, 1200, 10, dtype = int)) + [None],
'max_features': ['auto', 'sqrt','log2', None],
'min_samples_leaf': [4, 6, 8, 12],
'min_samples_split': [5, 7, 10, 14],
'n_estimators': list(np.linspace(151, 1200, 10, dtype = int)) }
clf = RandomForestClassifier()
model = RandomizedSearchCV(estimator = clf, param_distributions = random_search, n_iter = 80,
cv = 4, verbose= 5, random_state= 101, n_jobs = -1)
 model.fit(X,Y)
 model.best_params_
predicted = clf.predict(test )<save_to_csv>
random_search = {'criterion': ['entropy', 'gini'],
'max_depth': list(np.linspace(10, 1200, 10, dtype = int)) ,
'max_features': ['auto', 'sqrt','log2', None],
'min_samples_leaf': [4, 6, 8, 12],
'min_samples_split': [5, 7, 10, 14],
'n_estimators': list(np.linspace(151, 1200, 10, dtype = int)) }
clf = XGBClassifier()
model = RandomizedSearchCV(estimator = clf, param_distributions = random_search, n_iter = 80,
cv = 4, verbose= 5, random_state= 101, n_jobs = -1)
 model.fit(X,Y)
 model.best_params_
output = pd.DataFrame(predicted,columns = ['Outcome'])
 test = pd.read_csv('.. /input/test.csv')
 output['Id'] = test['Id']
output[['Id','Outcome']].to_csv('submission_cloudy10.csv', index = False)
 output.head()<set_options>
random_search = {'dual': [False],
'solver' : ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'],
'max_iter': list(np.linspace(10, 50000, 100, dtype = int)) }
clf = LogisticRegression()
model = RandomizedSearchCV(estimator = clf, param_distributions = random_search, n_iter = 80,
cv = 4, verbose= 5, random_state= 101, n_jobs = -1)
 model.fit(X,Y)
 model.best_params_
%matplotlib inline<load_from_csv>
models = [
LogisticRegression(solver= 'sag', max_iter = 15024, dual= False),
RandomForestClassifier(n_estimators = 1200, min_samples_split = 5, min_samples_leaf = 4, max_features = None, max_depth = 1200, criterion = 'entropy'),
XGBClassifier(n_estimators = 151, min_samples_split = 10, min_samples_leaf = 6,
max_features = 'auto', max_depth = 10, criterion = 'gini')
 ]
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
S_train, S_test = stacking(models,
X_Train, Y_Train, Test_Data,
regression=False,
mode='oof_pred_bag',
needs_proba=False,
save_dir=None,
metric=accuracy_score,
n_folds= 4,
stratified=True,
shuffle=True,
random_state=0,
verbose=2 )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<categorify>
model = XGBClassifier(n_estimators = 151, min_samples_split = 10, min_samples_leaf = 6,
max_features = 'auto', max_depth = 10, criterion = 'gini')
 model = model.fit(S_train, Y_Train)
 Y_Pred = model.predict(S_test)
imputer = Imputer()
train2 =train[['Pregnancies', 'Glucose', 'BloodPressure', 'SkinThickness',
'Insulin', 'BMI', 'DiabetesPedigreeFunction', 'Age',]].replace(0, numpy.NaN)
 train2["Outcome"]= train["Outcome"]
transformed_train = imputer.fit_transform(train2)
 print(numpy.isnan(transformed_train ).sum())
 train2.BMI.plot()
print(train2.shape)
<count_missing_values>
warnings.simplefilter('ignore')
 clf1 = KNeighborsClassifier(n_neighbors=1)
 clf2 = RandomForestClassifier(random_state=1)
 clf3 = GaussianNB()
lr = LogisticRegression()
sclf = StackingClassifier(classifiers=[clf1, clf2, clf3],
use_probas=True,
average_probas=False,
meta_classifier=lr)
 print('3-fold cross validation:
')
 for clf, label in zip([clf1, clf2, clf3, sclf],
['KNN',
'Random Forest',
'Naive Bayes',
'StackingClassifier']):
scores = model_selection.cross_val_score(clf, X, Y,
cv=3, scoring='accuracy')
 print("Accuracy: %0.2f(+/- %0.2f)[%s]"
%(scores.mean() , scores.std() , label))
train.isnull().sum()<count_missing_values>
sclf.fit(X_Train, Y_Train)
 Y_Pred = sclf.predict(X_Test)
 sclf.score(X_Test,Y_Test )
test.isnull().sum()<count_missing_values>
X_Train2, X_Val, Y_Train2, Y_Val = train_test_split(X_Train, Y_Train, test_size = 0.30, random_state = 101)
 model1 = tree.DecisionTreeClassifier()
model1.fit(X_Train2,Y_Train2)
 val_pred1=model1.predict(X_Val)
 test_pred1=model1.predict(X_Test)
 val_pred1=pd.DataFrame(val_pred1)
 test_pred1=pd.DataFrame(test_pred1)
 model2 = LogisticRegression()
model2.fit(X_Train2,Y_Train2)
 val_pred2=model2.predict(X_Val)
 test_pred2=model2.predict(X_Test)
 val_pred2=pd.DataFrame(val_pred2)
 test_pred2=pd.DataFrame(test_pred2)
 df_val=pd.concat([pd.DataFrame(X_Val), val_pred1,val_pred2],axis=1)
 df_test=pd.concat([pd.DataFrame(X_Test), test_pred1,test_pred2],axis=1)
 model = LogisticRegression()
model.fit(df_val,Y_Val)
 model.score(df_test,Y_Test )
test.isnull().sum()<count_values>
model = BaggingClassifier(tree.DecisionTreeClassifier(random_state=1))
model.fit(X_Train,Y_Train)
 model.score(X_Test,Y_Test )
train.Outcome.value_counts()<prepare_x_and_y>
model = AdaBoostClassifier()
model.fit(X_Train,Y_Train)
 model.score(X_Test,Y_Test )
X = train2.drop(['Outcome'], axis = 1)
 imputer = Imputer()
transformed_X = imputer.fit_transform(X)
 X_with_null = train.drop(['Outcome'], axis = 1)
 y = train2.Outcome
y_with_null = train.Outcome<compute_train_metric>
model= GradientBoostingClassifier(learning_rate=0.01,random_state=1)
 model.fit(X_Train,Y_Train)
 model.score(X_Test,Y_Test )
X_validate = X[:200]
Y_validate = y[:200]
params = {'learning_rate':0.1}
model = XGBClassifier(n_estimators=500, **params)
 kfold = KFold(n_splits=10, random_state=7)
 print("dropped nulls")
 print(cross_val_score(model, transformed_X, y,cv=kfold))<train_model>
model=xgb.XGBClassifier(random_state=1,learning_rate=0.01)
 model.fit(X_Train,Y_Train)
 model.score(X_Test,Y_Test )
X_validate_null = X_with_null[:200]
Y_validate_null = y_with_null[:200]
params = {'learning_rate':0.1}
model = XGBClassifier(n_estimators=500, **params)
 kfold = KFold(n_splits=10, random_state=7)
 print("Raw data")
 print(cross_val_score(model, X_validate_null, Y_validate_null,cv=kfold))<prepare_output>
train_data=lgb.Dataset(X_Train,Y_Train)
 params = {'learning_rate':0.2}
model= lgb.train(params, train_data, 100)
 y_pred=model.predict(X_Test)
 preds = np.where(y_pred > 0.5, 1, 0)
 print(confusion_matrix(Y_Test, preds))
print(classification_report(Y_Test, preds))
XX = pd.DataFrame(data=transformed_X)
 XX.columns = ['Pregnancies', 'Glucose', 'BloodPressure', 'SkinThickness',
'Insulin', 'BMI', 'DiabetesPedigreeFunction', 'Age']
XX['Id'] = train['Id']
XX.head()<train_model>
params = {'learning_rate':0.01}
clf = XGBClassifier(n_estimators=100, **params)
 clf.fit(XX,y )<predict_on_test>
def fit_model(trainX, trainy):
model = Sequential()
model.add(Dense(14, input_dim= 10, activation='relu'))
model.add(Dense(50))
model.add(Dense(10, activation='relu'))
model.add(Dense(2, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.fit(trainX, trainy, epochs=500, verbose=0)
 return model
def define_stacked_model(members):
for i in range(len(members)) :
model = members[i]
for layer in model.layers:
layer.trainable = False
layer.name = 'ensemble_' + str(i+1)+ '_' + layer.name
ensemble_visible = [model.input for model in members]
ensemble_outputs = [model.output for model in members]
merge = concatenate(ensemble_outputs)
 hidden = Dense(10, activation='relu' )(merge)
 output = Dense(2, activation='softmax' )(hidden)
 model = Model(inputs=ensemble_visible, outputs=output)
 plot_model(model, show_shapes=True, to_file='model_graph.png')
 model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
def fit_stacked_model(model, inputX, inputy):
X = [inputX for _ in range(len(model.input)) ]
model.fit(X, inputy, epochs=300, verbose=0)
 def predict_stacked_model(model, inputX):
X = [inputX for _ in range(len(model.input)) ]
return model.predict(X, verbose=0)
 BinY_Train = to_categorical(Y_Train)
 n_members = 5
members = []
for i in range(n_members):
members.append(fit_model(X_Train, BinY_Train))
print('Created %d models' % len(members))
stacked_model = define_stacked_model(members )
predicted = clf.predict(test)
<save_to_csv>
BinY_Test = to_categorical(Y_Test)
 fit_stacked_model(stacked_model, X_Test, BinY_Test)
 yhat = predict_stacked_model(stacked_model, X_Test)
 yhat = argmax(yhat, axis=1)
 acc = accuracy_score(Y_Test, yhat)
 print('Stacked Test Accuracy: %.3f' % acc )
output = pd.DataFrame(predicted,columns = ['Outcome'])
 test = pd.read_csv('.. /input/test.csv')
 output['Id'] = test['Id']
output[['Id','Outcome']].to_csv('johnnybgood2.csv', index = False)
 output.head()<set_options>
n_members = 7
members = []
for i in range(n_members):
members.append(fit_model(X_Train, BinY_Train))
print('Created %d models' % len(members))
stacked_model = define_stacked_model(members)
 fit_stacked_model(stacked_model, X_Test, BinY_Test)
 yhat = predict_stacked_model(stacked_model, Test_Data)
 yhat = argmax(yhat, axis=1 )
<load_from_csv><EOS> 
submission = pd.DataFrame({
"PassengerId": df2["PassengerId"],
"Survived": yhat
})
 submission.to_csv('titanic.csv', index=False)
 submission.head()