Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_variables>
warnings.filterwarnings("ignore")
 sns.set_style('whitegrid')
 sns.set_context("talk")
 %matplotlib inline
def reorder(old_list,order):
new_list = []
for i in order:
new_list.append(old_list[i])
 return new_list
np.random.seed(seed=42)
 indices = np.arange(len(fruit_labels))
np.random.shuffle(indices)
 indices = indices.tolist()
fruit_labels = reorder(fruit_labels,indices)
 fruit_images = reorder(fruit_images,indices)
 image_array = np.array(fruit_images)
 label_array = np.array(fruit_labels )<categorify>
train_df = pd.read_csv('.. /input/titanic/train.csv')
 test_df = pd.read_csv('.. /input/titanic/test.csv' )
label_array=to_categorical(label_array,3 )<split>
all_df = pd.concat([train_df, test_df], axis=0, sort=False, ignore_index=True )
X_train, X_test, y_train, y_test = train_test_split(image_array,label_array, test_size=0.2)
 X_train, X_val, y_train, y_val=train_test_split(X_train, y_train, test_size=0.3 )<choose_model_class>
all_df['Title'] = all_df['Name'].apply(lambda name: name.split(',')[1].strip().split('.')[0] )
model = Sequential([
Conv2D(32,(3,3), activation='relu', input_shape=(50,50,3),padding='same'),
MaxPooling2D(2, 2),
Conv2D(64,(3,3), activation='relu',padding='same'),
MaxPooling2D(2,2),
Conv2D(128,(3,3), activation='relu',padding='same'),
MaxPooling2D(2,2),
Flatten() ,
Dropout(0.50),
Dense(128, activation='relu'),
Dense(3, activation='softmax')
 ])
<choose_model_class>
newtitles={
"Capt": "Officer",
"Col": "Officer",
"Major": "Officer",
"Jonkheer": "Royalty",
"Don": "Royalty",
"Sir" : "Royalty",
"Dr": "Officer",
"Rev": "Officer",
"the Countess":"Royalty",
"Dona": "Royalty",
"Mme": "Mrs",
"Mlle": "Miss",
"Ms": "Mrs",
"Mr" : "Mr",
"Mrs" : "Mrs",
"Miss" : "Miss",
"Master" : "Master",
"Lady" : "Royalty"}
l=[25]
for i in range(len(l)) :
model = Sequential([
Conv2D(32,(3,3), activation='relu', input_shape=(50,50,3),padding='same'),
MaxPooling2D(2, 2),
Conv2D(64,(3,3), activation='relu',padding='same'),
MaxPooling2D(2,2),
Conv2D(128,(3,3), activation='relu',padding='same'),
MaxPooling2D(2,2),
Flatten() ,
Dropout(0.50),
Dense(128, activation='relu'),
Dense(3, activation='softmax')
 ])
 model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
 model.fit(X_train,y_train,epochs=l[i],verbose=1)
 modelsub=model.predict(X_val,batch_size=None, verbose=1)
 c=0
f=0
for i in range(modelsub.shape[0]):
if np.argmax(modelsub[i])==np.argmax(y_val[i]):
c+=1
else:
f+=1
accuracy=c/(c+f)
 print('Test Accuracy:',accuracy*100)
<predict_on_test>
all_df['Title'].update(all_df['Title'].map(newtitles))
modelsub=model.predict(X_test,batch_size=None, verbose=1)
 c=0
f=0
for i in range(modelsub.shape[0]):
if np.argmax(modelsub[i])==np.argmax(y_test[i]):
c+=1
else:
f+=1
accuracy=c/(c+f)
 print('Test Accuracy:',accuracy*100 )<load_from_csv>
all_df['Deck'] = all_df['Cabin'].apply(lambda cabin: cabin[0] if pd.notnull(cabin)else 'N' )
sub_df=pd.read_csv('.. /input/sample_submission_q2.csv')
 test_id = sub_df['ID']
test_id=test_id.astype(str)
 test_path='.. /input/test/Test/'
test = get_images(ids=test_id, filepath=test_path)
<save_to_csv>
all_df['Alone'] =(all_df['Parch'].apply(lambda value: not(value)) & all_df['SibSp'].apply(lambda value: not(value)) ).astype('int' )
l=[]
iid=np.arange(1,113,1)
 c=model.predict(test,batch_size=None, verbose=1)
 for i in range(len(c)) :
l.append(np.argmax(c[i]))
df=pd.DataFrame({'ID':iid,'Category':l
},index=iid)
 print(l)
 df.to_csv("q2.csv",index=False )<import_modules>
all_df['Relatives'] = all_df['SibSp'] + all_df['Parch']
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt<compute_train_metric>
all_df[all_df['Embarked'].isnull() ]
def classification_reports(classifier,features,labels):
size_data = len(labels)
 count_class_1 = sum(labels)
 count_class_0 = size_data - count_class_1
print(' class 1 : ', count_class_1)
 print(' class 0 : ', count_class_0)
 y = classifier.predict(features)
 fpr, tpr, thresholds = metr.roc_curve(labels, y)
 print("Confusion Matrix:
",metr.confusion_matrix(labels,y))
score=metr.accuracy_score(labels,y)
 print("Score: ",score)
 auc=metr.roc_auc_score(labels,y)
 print("AUC: ",auc)
 print(metr.classification_report(labels,y))
plt.figure()
plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
 Legend = "FirstStage"
plt.plot(fpr, tpr,color='darkorange')
 plt.show()
return fpr,tpr,score<normalization>
sort_embarked_features = ['Cabin', 'Sex', 'Pclass', 'Alone', 'Fare']
sorted_df = all_df.sort_values(by=sort_embarked_features)
 sorted_df[(sorted_df['Survived'] == 1)& 
(sorted_df['Sex'] == 'female')& 
(sorted_df['Pclass'] == 1)& 
(sorted_df['Alone'] == 1)].sort_values(by=sort_embarked_features)[sort_embarked_features+['Embarked']].head()
def pre_process(features):
del features['Name']
del features['PlayerID']
x = np.isfinite(features)
 features[(x == False)&(features > 0)] = 10000000
features[(x == False)&(features < 0)] = -10000000
imputer = Imputer(missing_values='NaN', strategy='mean', axis=0 ).fit(features)
 features= imputer.transform(features)
 features = normalize(features, axis=0,norm='max')
 return features<concatenate>
all_df['Embarked'].fillna(value='C', inplace=True )
def pre_process_pca(features,test):
Data = np.concatenate(( features,test))
Data = PCA(n_components=.95 ).fit_transform(Data)
 return Data[0:900,:],Data[900:1340,:]<normalization>
all_df[all_df['Fare'].isnull() ]
def pre_process_manifold(features,test):
n_neighbors = 15
Data = np.concatenate(( features, test))
Data = manifold.Isomap(n_neighbors, n_components=15 ).fit_transform(Data)
 return Data[0:900,:],Data[900:1340,:]<import_modules>
sort_fare_features = ['Sex', 'Pclass', 'Title', 'Deck', 'Embarked', 'Alone', 'Age']
sorted_df = all_df.sort_values(by=sort_embarked_features)
 aux = sorted_df[(sorted_df['Alone'] == 1)& 
(sorted_df['Sex'] == 'male')& 
(sorted_df['Pclass'] == 1)& 
(sorted_df['Embarked'] == 'S')& 
(sorted_df['Deck'] == 'N')& 
(sorted_df['Title'] == 'Mr')& 
(( sorted_df['Age'] >= 55)&(sorted_df['Age'] <= 65)) ]
aux
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn import svm
from sklearn import ensemble
from sklearn import naive_bayes
from sklearn import neural_network
from sklearn import neighbors
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.metrics import accuracy_score<load_from_csv>
all_df['Fare'].fillna(value=aux['Fare'].mean() , inplace=True )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<define_variables>
def impute_num(cols, avg, std):
try:
avg_value = avg.loc[tuple(cols)][0]
except Exception as e:
print(f'It is not possible to find an average value for this combination of features values:
{cols}')
 return np.nan
try:
std_value = std.loc[tuple(cols)][0]
except Exception as e:
std_value = 0
finally:
if pd.isnull(std_value):
std_value = 0
while True:
value = np.random.randint(avg_value-std_value, avg_value+std_value+1)
 if value >= 0:
break
return round(value, 0 )
labels = train.TARGET_5Yrs
features = train.iloc[:,0:21] 
<categorify>
group_age_features = ['Title','Relatives','Parch','SibSp','Deck','Pclass','Embarked','Sex','Alone']
features = pre_process(features)
 test = pre_process(test )<choose_model_class>
stat_age = all_df.pivot_table(values='Age', index=group_age_features, aggfunc=['mean','std'] ).round(2 )
classifier1 = neural_network.MLPClassifier(hidden_layer_sizes=(200,10),random_state=0)
 classifier2 = naive_bayes.GaussianNB()
classifier3 =neighbors.KNeighborsClassifier(n_neighbors=1)
 classifier4= svm.NuSVC(kernel='rbf',random_state=0)
 classifier5 = neural_network.MLPClassifier(hidden_layer_sizes=(20,5),random_state=0)
 classifier6 = svm.SVC(kernel='rbf', C=.5,random_state=0)
 classifier7 = ensemble.AdaBoostClassifier(n_estimators=60,learning_rate=.3,random_state=0)
 classifier8 = ensemble.GradientBoostingClassifier(n_estimators=20,max_depth=30,random_state=0)
 classifier9 = ensemble.RandomForestClassifier(n_estimators=60,random_state=0)
 classifier10 = ensemble.ExtraTreesClassifier(n_estimators=10,random_state=0)
 classifier11 = ensemble.BaggingClassifier(n_estimators=10,random_state=0)
 classifier13 = neighbors.KNeighborsClassifier(n_neighbors=3)
 classifier14 = QuadraticDiscriminantAnalysis()<train_model>
comp = pd.DataFrame([all_df[~all_df['Age'].isnull() ]['Age'], ages2, ages1] ).T
comp.columns = ['Real Age', 'Predicted Age $(\mu=\mu_{Age}$, $\sigma=\sigma_{Age})$', 'Predicted Age $(\mu=\mu_{Age}$, $\sigma=0)$']
comp.head(10 )
classifier12 = ensemble.IsolationForest(n_estimators=100,bootstrap="true",contamination=0.1,random_state=0)
 classifier12.fit(features, labels)
 result12 = classifier12.predict(test)
 outlier = classifier12.predict(features)
 delfeatures = np.where(outlier == -1 )<feature_engineering>
pd.DataFrame(data=[mean_squared_error(all_df[~all_df['Age'].isnull() ]['Age'], ages1),
mean_squared_error(all_df[~all_df['Age'].isnull() ]['Age'], ages2)],
index = ['$\mu=\mu_{Age}, \sigma=0$', '$\mu=\mu_{Age}, \sigma=\sigma_{Age}$'],
columns=['Mean Square Error'] ).round(1 )
for i in np.int32(delfeatures):
features[i,:] = np.sqrt(features[i,:] )<choose_model_class>
group_features = ['Title','Pclass','Embarked','Sex']
classifier = ensemble.VotingClassifier\ 
(estimators=[('mlp',classifier1),('svm',classifier4),('knn3',classifier13), 
('knn1',classifier3),('nb',classifier2),('mlpmini', classifier5), 
('svm.5',classifier6),('ada',classifier7),('gb',classifier8), 
('rf', classifier9),('et',classifier10),('bag',classifier11), 
('quad',classifier14)], voting='hard',flatten_transform='true',
weights=[2,2,1,1,1,1,1,1,1,1,1,1,1] 
)
 classifier.fit(features, labels)
 result = classifier.predict(test)
 result[result==-1]=0<compute_train_metric>
stat_age = all_df.pivot_table(values='Age', index=group_features, aggfunc=['mean','std'] ).round(2 )
scores = cross_val_score(classifier, features, labels, cv=10,scoring='f1')
 classification_reports(classifier,features,labels)
 print("Accuracy: %0.2f(+/- %0.2f)" %(scores.mean() , scores.std() * 2))
print(scores)
<save_to_csv>
def sort_remap(crosstab_df, key):
alives = list(crosstab_df[crosstab_df['Survived']==1][key])
 deads = list(crosstab_df[crosstab_df['Survived']==0][key])
 alives = list(set(deads)- set(alives)) + alives
sorted_map = {key:value for value, key in enumerate(alives)}
return sorted_map
cols = { 'PlayerID': [i+901 for i in range(440)] , 'TARGET_5Yrs': result }
submission = pd.DataFrame(cols)
 submission.to_csv("subm.csv", index=False )<import_modules>
kdis = KBinsDiscretizer(n_bins=8, encode='ordinal', strategy='uniform' ).fit(all_df[['Age', 'Fare']] )
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import Imputer
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.preprocessing import StandardScaler
from statsmodels.compat import pandas as pd
import pandas as pd
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn import svm, preprocessing
from sklearn.decomposition import PCA<prepare_x_and_y>
cat_age_fare = kdis.transform(all_df[['Age', 'Fare']] )
test = pd.read_csv(".. /input/test.csv")
 test_x = test.iloc[:, 2:].values
imp = Imputer(missing_values = 'NaN', strategy = 'mean', axis = 0)
 imp=imp.fit(test_x)
 test_x = imp.transform(test_x)
 dataset = pd.read_csv(".. /input/train.csv")
 print(dataset.info())
 X = dataset.iloc[:, 2:-1].values
imputer = Imputer(missing_values = 'NaN', strategy = 'mean', axis = 0)
 input_dim = X.shape[1]
imputer = imputer.fit(X)
 X = imputer.transform(X)
 y = dataset.iloc[:, 21].values
sc = StandardScaler()
X_train = sc.fit_transform(X)
 x_test = sc.transform(test_x)
 Xtrain = preprocessing.normalize(X_train, norm='l2')
 Xtest = preprocessing.normalize(x_test, norm='l2')
 print(X)
 print(y)
 print(test_x )<save_to_csv>
all_df['Cat_Age'] = cat_age_fare[:,0].astype('int')
 all_df['Cat_Fare'] = cat_age_fare[:,1].astype('int' )
cols = { 'PlayerID': [i+901 for i in range(440)] , 'TARGET_5Yrs': [eclf1.predict([test_x[i]])[0] for i in range(440)] }
submission = pd.DataFrame(cols)
 print(submission)
 submission.to_csv("submission1.csv", index=False )<import_modules>
features_on_off = {'PassengerId':False,
'Survived':False,
'Pclass':True,
'Name':False,
'Sex':True,
'Age':False,
'SibSp':True,
'Parch':True,
'Ticket':False,
'Fare':False,
'Cabin':False,
'Embarked':True,
'Title':True,
'Deck':True,
'Alone':True,
'Relatives':True,
'Cat_Age':True,
'Cat_Fare':True}
import numpy as np
import pandas as pd<load_from_csv>
poly = PolynomialFeatures(degree=2, include_bias=False ).fit(all_df[features_on])
 poly_features = poly.transform(all_df[features_on])
 poly_df = pd.DataFrame(data=poly_features, columns=poly.get_feature_names(features_on))
df = pd.read_csv('/kaggle/input/data-driven-business-analytics/train.csv', index_col=0)
 df.head()<groupby>
std_scaler = StandardScaler().fit(poly_df)
 scaled_features = std_scaler.transform(poly_df)
 scaled_df = pd.DataFrame(data=scaled_features, columns=poly_df.columns )
mean_price = df['Preis'].mean()
mean_price<load_from_csv>
new_df = scaled_df.copy()
new_df['Survived'] = all_df['Survived']
df_test = pd.read_csv('/kaggle/input/data-driven-business-analytics/test.csv', index_col=0)
 df_test.head()<predict_on_test>
X_train = scaled_df.loc[range(train_df.shape[0])]
y_train = all_df.loc[range(train_df.shape[0]), 'Survived']
X_test = scaled_df.loc[range(train_df.shape[0], train_df.shape[0]+test_df.shape[0])]
y_test = pd.read_csv('.. /input/true-labels/submission_true.csv')['Survived']
predictions = [mean_price] * 322
predictions[:5]<feature_engineering>
log_reg = LogisticRegressionCV(Cs=1000, cv=5, refit=True, random_state=101, max_iter=200 ).fit(X_train, y_train )
df_test['Preis'] = predictions
df_test.head()<save_to_csv>
print(f'Accuracy on the training set: {100*log_reg.score(X_train, y_train):.1f}%' )
df_submission.to_csv('./submission.csv', index=False )<load_from_csv>
print(f'Accuracy on the test set: {100*log_reg.score(X_test, y_test):.1f}%' )
train_data = pd.read_csv("/kaggle/input/figure-out-the-formula-dogfooding/train.csv")
 train_data.head()<load_from_csv>
p = np.linspace(0.01, 1, 50)
 error_rate = {'train':[], 'test':[]}
for p_value in p:
prob = log_reg.predict_proba(X_train)
 y_pred = np.apply_along_axis(lambda pair: 1 if pair[1] > p_value else 0, 1, prob)
 error_rate['train'].append(mean_squared_error(y_train, y_pred))
prob = log_reg.predict_proba(X_test)
 y_pred = np.apply_along_axis(lambda pair: 1 if pair[1] > p_value else 0, 1, prob)
 error_rate['test'].append(mean_squared_error(y_test, y_pred))
test_data = pd.read_csv("/kaggle/input/figure-out-the-formula-dogfooding/test.csv")
 test_data.head()<prepare_x_and_y>
best_p = p[np.array(error_rate['test'] ).argmin() ]
knn = KNeighborsRegressor(n_neighbors=3)
 train_data = train_data.drop("Id", axis=1)
 x = train_data.drop("Expected", axis=1)
 y = train_data["Expected"]
knn.fit(x, y)
 print("Done training" )<predict_on_test>
print(f'In this particular case, the best probability limit is around {best_p:.2}.This means that if the model provides a survival value greater than {best_p:.2}, we accept that the passenger will survive or die otherwise.' )
solutions = []
for index, row in test_data.iterrows() :
row_id = row["Id"]
variables = row["a":"h"]
prediction = knn.predict([variables])
 print("Prediction is " + str(prediction))
solutions.append([row_id, prediction[0]])
 solutions<save_to_csv>
prob = log_reg.predict_proba(X_train)
 y_pred_train = np.apply_along_axis(lambda pair: 1 if pair[1] > best_p else 0, 1, prob)
 print(f'Accuracy on the training set with the best choice of p: {100*np.mean(y_train == y_pred_train):.1f}%' )
submission = pd.DataFrame(solutions, columns=["Id", "Expected"])
 submission.to_csv("submission.csv", index=False )<set_options>
prob = log_reg.predict_proba(X_test)
 y_pred_test = np.apply_along_axis(lambda pair: 1 if pair[1] > best_p else 0, 1, prob)
 print(f'Accuracy on the testing set with the best choice of p: {100*np.mean(y_pred_test == y_test):.1f}%' )
%matplotlib inline<load_from_csv>
print(classification_report(y_pred_train, y_train))
df = pd.read_csv('.. /input/train.csv')
 df.head()<load_from_csv>
print(classification_report(y_pred_test, y_test))
class ImageDataset(Dataset):
def __init__(self, csv_file, root_dir, transform=None):
self.df = pd.read_csv(csv_file)
 self.dir = root_dir
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
img_path = os.path.join(self.dir, self.df['Id'][index])
 image = Image.open(img_path ).convert('RGB')
 label = np.array(self.df['Category'][index])
 if self.transform:
image = self.transform(image)
 sample =(image, label)
 return sample 
<train_model>
perm = PermutationImportance(log_reg ).fit(X_train, y_train)
 show_weights(perm, feature_names = list(X_train.columns))
gpu = torch.cuda.is_available()
if gpu:
print("CUDA is available.Training on GPU")
 else:
print("CUDA unavailable.Training on CPU" )<choose_model_class>
weights = pd.DataFrame(data=log_reg.coef_[0], index=scaled_df.columns, columns=['Weights'])
 weights.loc['Intercept'] = log_reg.intercept_
weights.sort_values('Weights', ascending=False, inplace=True)
 weights.head(15 )
model = models.vgg19(pretrained=True)
 model.classifier = nn.Sequential(nn.Linear(512 * 7 * 7, 4096),
nn.LeakyReLU() ,
nn.Dropout() ,
nn.Linear(4096, 512),
nn.LeakyReLU() ,
nn.Dropout() ,
nn.Linear(512, 67), 
)
 model.cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters() , lr=0.001, momentum=0.9)
 scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1 )<train_model>
output = pd.DataFrame({'PassengerId': test_df['PassengerId'], 'Survived': y_pred_test.astype('int')} )
<train_model><EOS> 
output.to_csv('submission.csv', index=False )