Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_disk>
import numpy as np
import pandas as pd
import seaborn as sns
from scipy import stats
import matplotlib.pyplot as plt
import torch
from torch import nn
import torch.optim
from torch.nn import functional as F
from torch.utils.data import TensorDataset, DataLoader
from torch.utils.data.sampler import SubsetRandomSampler
from sklearn.ensemble import RandomForestClassifier
with open('.. /input/test.json')as test_data:
test_data = json.load(test_data )<define_variables>
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 train_data.head()
features_test = [x['ingredients'] for x in test_data]<define_variables>
test_data = pd.read_csv('/kaggle/input/titanic/test.csv')
 test_data.head()
features_list_test = []
for feature in features_test:
single_item = ''
for item in feature:
single_item = single_item + ' ' + item
single_item = single_item[1:]
features_list_test.append(single_item )<feature_engineering>
women = train_data.loc[train_data.Sex=='female']["Survived"]
rate_women = sum(women)/ len(women)
 F"% of women who survived: {rate_women}"
test_features = vectorizor.transform(features_list_test )<train_model>
men = train_data.loc[train_data.Sex=='male']['Survived']
men_rate = sum(men)/ len(men)
 F"% of men who survived: {men_rate}"
model= LogisticRegression()
model.fit(X_features,y )<compute_test_metric>
fare_mean_1st = train_data[train_data["Pclass"]==1].Fare.mean()
fare_mean_2nd = train_data[train_data["Pclass"]==2].Fare.mean()
fare_mean_3rd = train_data[train_data["Pclass"]==3].Fare.mean()
F"Average cost of tickets for 1st, snd, 3rd classes: \
{fare_mean_1st} || {fare_mean_2nd} || {fare_mean_3rd}"
model.score(X_features, y )<load_from_disk>
woman_survived_1st = len(train_data[(train_data["Sex"]=="female")&(train_data["Survived"]==1)&(train_data["Pclass"]==1)].index)/ len(train_data[(train_data["Sex"]=="female")&(train_data["Pclass"]==1)].index)
 woman_survived_2nd = len(train_data[(train_data["Sex"]=="female")&(train_data["Survived"]==1)&(train_data["Pclass"]==2)].index)/ len(train_data[(train_data["Sex"]=="female")&(train_data["Pclass"]==2)].index)
 woman_survived_3rd = len(train_data[(train_data["Sex"]=="female")&(train_data["Survived"]==1)&(train_data["Pclass"]==3)].index)/ len(train_data[(train_data["Sex"]=="female")&(train_data["Pclass"]==3)].index)
 F"Rate of Survival for women in different classes: {woman_survived_1st} || {woman_survived_2nd} || {woman_survived_3rd}"
df = pd.read_json('.. /input/test.json' )<predict_on_test>
woman_survived_1st = len(train_data[(train_data["Sex"]=="male")&(train_data["Survived"]==1)&(train_data["Pclass"]==1)].index)/ len(train_data[(train_data["Sex"]=="male")&(train_data["Pclass"]==1)].index)
 woman_survived_2nd = len(train_data[(train_data["Sex"]=="male")&(train_data["Survived"]==1)&(train_data["Pclass"]==2)].index)/ len(train_data[(train_data["Sex"]=="male")&(train_data["Pclass"]==2)].index)
 woman_survived_3rd = len(train_data[(train_data["Sex"]=="male")&(train_data["Survived"]==1)&(train_data["Pclass"]==3)].index)/ len(train_data[(train_data["Sex"]=="male")&(train_data["Pclass"]==3)].index)
 F"Rate of Survival for men in different classes: {woman_survived_1st} || {woman_survived_2nd} || {woman_survived_3rd}"
df['cuisine'] = le.inverse_transform(model.predict(test_features))<save_to_csv>
train_data.isnull().sum()
df= df[['id', 'cuisine']]
df.to_csv('submit.csv', index=False )<load_from_csv>
test_data.isnull().sum()
train = pd.read_json('.. /input/train.json', orient='columns')
 test = pd.read_json('.. /input/test.json', orient='columns')
 sample_submission = pd.read_csv(".. /input/sample_submission.csv" )<feature_engineering>
X = train_data.drop(['PassengerId', 'Name', 'Ticket', 'Cabin', 'Embarked'], axis=1)
 X_test = test_data.drop(['PassengerId','Name', 'Ticket', 'Cabin', 'Embarked'], axis=1 )
def sub_space(x):
temp_value = list()
for i in x:
temp_value.append(re.sub(r'[^0-9a-zA-Z]+','_',i.lower()))
return temp_value
train['ingredients_new'] = train['ingredients'].apply(sub_space)
 test['ingredients_new'] = test['ingredients'].apply(sub_space)
 def convert_list_to_sent(x):
return ' '.join(x)
 train['ingredient_sent'] = train['ingredients_new'].apply(convert_list_to_sent)
 test['ingredient_sent'] = test['ingredients_new'].apply(convert_list_to_sent)
 display(train.head())
 display(test.head())
<split>
X = pd.get_dummies(X)
 X_test = pd.get_dummies(X_test)
 X.fillna(X.mean() ,inplace=True)
 X_test.fillna(X_test.mean() ,inplace=True )
X_train, X_val, y_train, y_val = train_test_split(train['ingredient_sent'], train['cuisine'], test_size=0.33, random_state=42 )<categorify>
X.isnull().sum()
tfidf_vect = TfidfVectorizer(lowercase=True,binary=True)
 X_train_tfidf = tfidf_vect.fit_transform(X_train)
 X_val_tfidf = tfidf_vect.transform(X_val)
 X_test_tfidf = tfidf_vect.transform(test['ingredient_sent'])
<categorify>
X_test.isnull().sum()
lb = LabelEncoder()
y_train_encode = lb.fit_transform(y_train)
 y_val_encode = lb.transform(y_val)
 y_train_dummy = np_utils.to_categorical(y_train_encode)
 y_val_dummy = np_utils.to_categorical(y_val_encode )<choose_model_class>
features = ["Pclass", "Sex_female", "Age", "Fare", "SibSp", "Parch"]
y= X['Survived']
X = pd.DataFrame(X, columns = features)
 X_test = pd.DataFrame(X_test, columns = features)
 for col in features:
X[col] =(X[col] - X[col].mean())/ X[col].std()
X_test[col] =(X_test[col] - X_test[col].mean())/ X_test[col].std()
for col in features:
X[col] =(X[col] - X[col].min())/(X[col].max() - X[col].min())
 X_test[col] =(X_test[col] - X_test[col].min())/(X_test[col].max() - X_test[col].min())
input_shape = X_train_tfidf.shape[1]
def model_structure1() :
mdl = Sequential()
mdl.add(Dense(512, init='glorot_uniform', activation='relu',input_shape=(input_shape,)))
 mdl.add(Dropout(0.5))
mdl.add(Dense(128, init='glorot_uniform', activation='relu'))
mdl.add(Dropout(0.5))
mdl.add(Dense(20, activation='softmax'))
mdl.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy'])
 mdl.summary()
return mdl<choose_model_class>
model = RandomForestClassifier(n_estimators = 100, max_features='auto', criterion='entropy',max_depth=10)
 model.fit(X, y)
 predictions = model.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('random_forest_submission.csv', index=False)
 print("Your submission was successfully saved!")
 cv_scores = cross_val_score(model, X, y,
cv=5,
scoring='accuracy')
 print("Cross-validation accuracy: %f" % cv_scores.mean() )
print("Compile model...")
 estimator = KerasClassifier(build_fn=model_structure1, epochs=10, batch_size=128 )<train_model>
df = pd.read_csv("random_forest_submission.csv")
 df
history = estimator.fit(X_train_tfidf.toarray() , y_train_dummy,\
validation_data=(X_val_tfidf.toarray() ,y_val_dummy)) 
<predict_on_test>
X = X.to_numpy()
y = y.to_numpy().reshape(-1, 1)
 X_test = X_test.to_numpy()
mnb_train_prediction = estimator.predict_proba(X_train_tfidf.toarray())
 mnb_val_prediction = estimator.predict_proba(X_val_tfidf.toarray())
 mnb_tr_pred_value = estimator.predict(X_train_tfidf.toarray())
 mnb_val_pred_value = estimator.predict(X_val_tfidf.toarray())
 mnb_test_pred_value = estimator.predict(X_test_tfidf.toarray())
 print("=====")
 print("Log loss for TfIDFVectorizer features in MLP for Training set {}".format(log_loss(y_train_encode,mnb_train_prediction)))
 print("Log loss for TfIDFVectorizer features in MLP for validation set {}".format(log_loss(y_val_encode,mnb_val_prediction)))
 print("Accuracy for TfIDFVectorizer features in MLP for Training set {}".format(accuracy_score(y_train_encode,mnb_tr_pred_value)))
 print("Accuracy for TfIDFVectorizer features in MLP for validation set {}".format(accuracy_score(y_val_encode,mnb_val_pred_value)) )<categorify>
def batch_data(batch_size, input_data, target, test_data, train_type = "regression", val_size=0.1):

 if train_type == "regression":
target_tensor = torch.FloatTensor(target)
 elif train_type == "classification":
target_tensor = torch.LongTensor(target)
 target_tensor = target_tensor.squeeze()
input_tensor = torch.FloatTensor(input_data)
 test_tensor = torch.FloatTensor(test_data)
 train_dataset = TensorDataset(input_tensor, target_tensor)
 num_train = len(train_dataset)
 indicies = list(range(num_train))
np.random.shuffle(indicies)
 val_split = int(np.floor(val_size * num_train))
train_idx, val_idx = indicies[val_split:], indicies[:val_split]
train_sampler = SubsetRandomSampler(train_idx)
 val_sampler = SubsetRandomSampler(val_idx)
 train_loader = DataLoader(train_dataset, batch_size=batch_size, sampler = train_sampler)
 val_loader = DataLoader(train_dataset, batch_size=batch_size, sampler = val_sampler)
 test_loader = DataLoader(test_tensor, batch_size=len(test_data))
return train_loader, val_loader, test_loader
test_pred = list(lb.inverse_transform(mnb_test_pred_value))
print(test_pred )<prepare_output>
batch_size = 32
train_loader, val_loader, test_loader = batch_data(batch_size, X, y, X_test )
result = pd.DataFrame({'id':test['id'],'cuisine':test_pred})
 result.head() 
<save_to_csv>
class LinearRegression(nn.Module):
def __init__(self):
super().__init__()
self.fc1 = nn.Linear(6, 20)
 self.fc2 = nn.Linear(20, 1)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
x = self.fc1(x)
 x = self.sigmoid(x)
 x = self.fc2(x)
 return x
result.to_csv('submission.csv',index=False )<load_from_csv>
def init_weights(m):
if type(m)== nn.Linear:
torch.nn.init.xavier_uniform(m.weight)
 m.bias.data.fill_(0.01 )
sns.set_style("whitegrid")
 with open('.. /input/train.json', 'r')as f:
txt = f.read()
df = pd.DataFrame(json.loads(txt))
df.head()<feature_engineering>
linear_regression_model = LinearRegression()
linear_regression_model.apply(init_weights)
 print(linear_regression_model )
df['joined'] = df.ingredients.map(lambda x: ' '.join(x))
df_nb = df[['cuisine','joined']]
df_nb.head()<feature_engineering>
lr = 0.01
criterion = nn.MSELoss()
optimizer = torch.optim.SGD(linear_regression_model.parameters() , lr=lr, momentum=0.9)
 batch_size = 32
count_vect = CountVectorizer()
tfidf_transformer = TfidfTransformer()
X = count_vect.fit_transform(df_nb.joined)
 X = tfidf_transformer.fit_transform(X)
 X.shape<compute_train_metric>
def train_model(model, batch_size, epochs, cost_function, print_every = 100):
val_loss_min = np.Inf
for e in range(epochs):
val_loss = 0.0
train_loss = 0.0
model.train()
for inputs, labels in train_loader:
optimizer.zero_grad()
output = model(inputs)
 loss = cost_function(output, labels)
 loss.backward()
optimizer.step()
train_loss += loss.item() * batch_size
model.eval()
for inputs, labels in val_loader:
output = model(inputs)
 loss = cost_function(output, labels)
 val_loss += loss.item() * batch_size
train_loss = train_loss / len(train_loader.sampler)
 val_loss = val_loss / len(val_loader.sampler)
 if val_loss <= val_loss_min:
print('Validation loss decreased({:.6f} --> {:.6f} ).Saving model...'.format(
 val_loss_min, val_loss))
val_loss_min = val_loss
torch.save(model.state_dict() , 'model_linear.pt')
 if epochs % print_every == 0:
print('Epoch: {} \tTraining Loss: {:.6f} \tValidation Loss: {:.6f}'.format(e, train_loss, val_loss))
print("Best model with validation loss: {}".format(val_loss_min))
clf = MultinomialNB()
scores = cross_val_score(clf, X, df_nb.cuisine, cv=5)
 print('accuracy CV:',scores )<choose_model_class>
train_model(linear_regression_model, batch_size, epochs=3000, cost_function=criterion )
def simple_NN(input_shape, nodes_per=[60], hidden=0, out=2, act_out='softmax', act_hid='relu', drop=True, d_rate=0.1):

 model = Sequential()
model.add(Dense(nodes_per[0],activation=act_hid,input_shape=input_shape))
if drop:
model.add(Dropout(d_rate))
try:
if hidden != 0:
for i,j in zip(range(hidden), nodes_per[1:]):
model.add(Dense(j,activation=act_hid))
if drop:
model.add(Dropout(d_rate))
model.add(Dense(out,activation=act_out))
return(model)
 except:
print('Error in generating hidden layers')
 ch_dict = dict([(y,x)for x,y in enumerate(set(df_nb.cuisine)) ])
 y = np.array([ch_dict[x] for x in df_nb.cuisine])
 y = to_categorical(y)
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)
 print('creating and training model...')
 model = simple_NN(input_shape=(X.shape[1],), nodes_per=[100, 100], hidden=1, out=y.shape[1], drop=True)
 model.compile(loss='categorical_crossentropy', optimizer=Adam() , metrics=['acc'])
 early_stopping_monitor = EarlyStopping(patience=3)
 history = model.fit(X_train,y_train, epochs=1000, validation_data=(X_test, y_test), callbacks=[early_stopping_monitor], verbose=True)
 print('model trained')
<load_from_csv>
linear_regression_model.load_state_dict(torch.load('model_linear.pt'))
with open('.. /input/test.json', 'r')as f:
txt = f.read()
df_test = pd.DataFrame(json.loads(txt))
df_test['joined'] = df_test.ingredients.map(lambda x: ' '.join(x))
df_test = df_test.drop(['ingredients'], axis=1)
 df_test.head()<predict_on_test>
with torch.no_grad() :
for data in test_loader:
output = linear_regression_model(data)
 preds = torch.round(output)
 preds = preds.squeeze()
survived = preds.numpy()
dec_dict = dict([(x,y)for y,x in ch_dict.items() ])
 X_test = np.array(df_test.joined)
 X_test = count_vect.transform(X_test)
 X_test = tfidf_transformer.transform(X_test)
 preds = model.predict(X_test)
 y_test = [dec_dict[np.argmax(x)] for x in preds]
df_test['cuisine'] = y_test
df_test.head()<save_to_csv>
survived = survived.astype('int' )
df_test = df_test.drop('joined', axis=1)
 df_test.to_csv('result.csv', index=False)
 print('written to csv.' )<set_options>
submission = pd.DataFrame({'PassengerId': test_data['PassengerId'], 'Survived': survived})
 submission.to_csv('submission_regression.csv', index=False)
 print("Your submission was successfully saved!" )
% matplotlib inline<load_from_disk>
df = pd.read_csv("submission_regression.csv")
 df
train = pd.read_json(".. /input/train.json")
 test = pd.read_json(".. /input/test.json" )<categorify>
train_loader, val_loader, test_loader = batch_data(batch_size, X, y, X_test, train_type="classification", val_size=0.2 )
train["ingredients"] = [", ".join(ingredients)for ingredients in train.ingredients]
test["ingredients"] = [", ".join(ingredients)for ingredients in test.ingredients]
target_enc = LabelEncoder()
y = target_enc.fit_transform(train.cuisine )<string_transform>
class Clasification(nn.Module):
def __init__(self):
super().__init__()
self.fc1 = nn.Linear(6, 20)
 self.fc2 = nn.Linear(20, 2)
 def forward(self, x):
x = F.relu(self.fc1(x))
x = self.fc2(x)
 return x
def tokenize(text):
tokens = nltk.word_tokenize(text)
 stems = [PorterStemmer().stem(word)for word in tokens]
return(stems)
 tfidf = TfidfVectorizer(tokenizer=tokenize )<feature_engineering>
classification_model = Clasification()
classification_model.apply(init_weights)
 classification_model
X = tfidf.fit_transform(train.ingredients )<train_model>
lr = 0.01
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(classification_model.parameters() , lr=lr, momentum=0.9)
 batch_size = 64
model = make_pipeline(TfidfVectorizer() , LinearSVC(C = 0.5))
model.fit(train.ingredients, y )<choose_model_class>
train_model(classification_model, batch_size, 3000, criterion )
svd = TruncatedSVD(n_components=300 )<feature_engineering>
with torch.no_grad() :
for data in test_loader:
output = classification_model(data.float())
 _, preds = torch.max(output.data, 1)
 survived = preds.numpy()
X_proj = svd.fit_transform(X )<choose_model_class>
submission = pd.DataFrame({'PassengerId': test_data['PassengerId'], 'Survived': survived})
 submission.to_csv('submission_classification.csv', index=False)
 print("Your submission was successfully saved!" )
model = LinearSVC(C = 0.5)
<compute_test_metric>
df = pd.read_csv("submission_classification.csv")
 df.head()
cross_val_score(model, X_proj, y )<compute_test_metric>
X_valid = [next(iter(val_loader)) [0].numpy() ]
y_valid = next(iter(val_loader)) [1].numpy()
cross_val_score(model, train.ingredients, y )<predict_on_test>
from xgboost import XGBClassifier
from sklearn.metrics import mean_absolute_error
preds = model.predict(test.ingredients)
 preds = target_enc.inverse_transform(preds )<create_dataframe>
xg_model = XGBClassifier(learning_rate=0.05, n_estimators=800)
 xg_model.fit(X, y )
solution = pd.DataFrame({"id":test.id, "cuisine":preds} )<save_to_csv>
predictions = xg_model.predict(X_test)
 predictions
<set_options><EOS> 
output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('xg_boost_submission.csv', index=False)
 print("Your submission was successfully saved!" )