Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<string_transform>
pd.options.mode.chained_assignment = None
%matplotlib inline
warnings.filterwarnings('ignore' )
def data_cleaner(text):
text = text.lower()
text = remove_punctuation_numbers(text)
 lst = tokeniser(text)
 lst = remove_stop_words(lst)
 return ' '.join(lemmetizer(lst))<feature_engineering>
train_data = pd.read_csv(".. /input/titanic/train.csv", index_col="PassengerId")
 test_data = pd.read_csv(".. /input/titanic/test.csv", index_col="PassengerId" )
train_data['clean_text'] = train_data['text'].apply(data_cleaner )<categorify>
print(train_data["Pclass"].unique())
 train_data[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
train_data.president = train_data.president.map({'deKlerk':0,'Mandela':1,
'Mbeki':2, 'Motlanthe':3,
'Zuma': 4, 'Ramaphosa':5} )<split>
train_data.Name[1].split()
X = train_data.clean_text
y = train_data.president
X_train, X_test, y_train, y_test = train_test_split(X, y,
test_size=0.33,
random_state=0,
stratify=y )<not_enough_vertices>
train_data = train_data.assign(fname = train_data.Name.str.split("," ).str[0])
 train_data["title"] = pd.Series([i.split(",")[1].split(".")[0].strip() for i in train_data.Name], index=train_data.index )
vect = CountVectorizer(ngram_range=(1,2))<feature_engineering>
test_data = test_data.assign(fname = test_data.Name.str.split("," ).str[0])
 test_data["title"] = pd.Series([i.split(",")[1].split(".")[0].strip() for i in test_data.Name], index=test_data.index)
 train_data.drop("Name", axis=1, inplace=True)
 test_data.drop("Name", axis=1, inplace=True )
X_train_ = vect.fit_transform(X_train )<choose_model_class>
print(test_data.fname.nunique())
 print(test_data.title.nunique() )
log = LogisticRegressionCV(dual=False, penalty='l2', multi_class='multinomial' )<train_model>
train_data["title"] = train_data['title'].replace(other_titles, 'Other')
 train_data["title"] = train_data["title"].map({"Mr":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Master":2, "Other":3})
 test_data["title"] = test_data['title'].replace(other_titles, 'Other')
 test_data["title"] = test_data["title"].map({"Mr":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Master":2, "Other":3} )
log.fit(X_train_, y_train )<compute_train_metric>
print(train_data.title)
 print(test_data.title.isna().sum() )
print(accuracy_score(y_train, log.predict(X_train_)) )<compute_test_metric>
oh = OneHotEncoder(handle_unknown="ignore", sparse = False)
 train_data = train_data.join(pd.DataFrame(oh.fit_transform(train_data[["fname", "title"]]), index = train_data.index))
test_data = test_data.join(pd.DataFrame(oh.transform(test_data[["fname", "title"]]), index = test_data.index))
train_data.drop("fname", axis = 1, inplace = True)
 test_data.drop("fname", axis = 1, inplace = True )
print(accuracy_score(y_test, log.predict(vect.transform(X_test))))<load_from_csv>
print(train_data["Sex"].unique())
 train_data[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
test_data = pd.read_csv('.. /input/test.csv' )<feature_engineering>
interactions = train_data.assign(sex_class = train_data['Sex'] + "_" + train_data['Pclass'].astype("str"))
interactions[['sex_class', 'Survived']].groupby(['sex_class'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
test_data.text = test_data.text.apply(data_cleaner )<predict_on_test>
train_data = train_data.assign(sex_class = train_data['Sex'] + "_" + train_data['Pclass'].astype("str"))
test_data = test_data.assign(sex_class = test_data['Sex'] + "_" + test_data['Pclass'].astype("str"))
test_data['president'] = log.predict(vect.transform(test_data.text))<drop_column>
train_data = train_data.join(pd.get_dummies(train_data['Pclass'], prefix="Pclass"))
test_data = test_data.join(pd.get_dummies(test_data['Pclass'], prefix="Pclass"))
test_data.drop('text', axis=1, inplace=True, )<save_to_csv>
train_data["Sex"] = train_data["Sex"].map({"female":0, "male":1})
 test_data["Sex"] = test_data["Sex"].map({"female":0, "male":1} )
test_data.to_csv('Thapelo_log.csv',index=False )<import_modules>
train_data["sex_class"] = train_data["sex_class"].map({"female_1":0, "female_2":1, "female_3":2, "male_1":4, "male_2":5, "male_3":6})
 test_data["sex_class"] = test_data["sex_class"].map({"female_1":0, "female_2":1, "female_3":2, "male_1":4, "male_2":5, "male_3":6} )
from sklearn.preprocessing import LabelEncoder
from sklearn.linear_model import LogisticRegression<load_from_csv>
def find_similar_passengers(id, dataset):
subset = dataset[(dataset.title == dataset.title[id])& 
(dataset.Pclass == dataset.Pclass[id])]
if subset["Age"].mean() == "NaN":
subset = dataset[(dataset["sex_class"] == dataset.iloc[id]["sex_class"])]
if subset["Age"].mean() == "NaN":
subset = dataset[(dataset["sex"] == dataset.iloc[id]["sex"])]
age = subset["Age"].mean()
return age
train_users_game1_df = pd.read_csv(".. /input/ds2019uec-task2/train_users_game1.csv")
 train_users_game2_df = pd.read_csv(".. /input/ds2019uec-task2/train_users_game2.csv")
 test_users_game1_df = pd.read_csv(".. /input/ds2019uec-task2/test_users_game1.csv" )<categorify>
no_ages = train_data[train_data["Age"].isna() ].index
for pid in no_ages:
train_data.Age[pid] = find_similar_passengers(pid, train_data)
 no_ages_test = test_data[test_data["Age"].isna() ].index
for pid2 in no_ages_test:
test_data.Age[pid2] = find_similar_passengers(pid2, test_data )
train_user_encoder = LabelEncoder()
train_user_encoder.fit(np.concatenate([train_users_game1_df["user_id"], train_users_game2_df["user_id"]]))
print(train_user_encoder.classes_.shape)
 game1_encoder = LabelEncoder()
game1_encoder.fit(np.concatenate([train_users_game1_df["game_title"], test_users_game1_df["game_title"]]))
print(game1_encoder.classes_.shape )<categorify>
train_data["age_group"] = pd.cut(train_data["Age"], bins=[0,5,65,100], labels=[0,1,2] ).astype("int64")
 test_data["age_group"] = pd.cut(test_data["Age"], bins=[0,5,65,100], labels=[0,1,2] ).astype("int64" )
X = np.zeros(( 9393, 4155))
user_vec = train_user_encoder.transform(train_users_game1_df["user_id"])
 game_vec = game1_encoder.transform(train_users_game1_df["game_title"])
 for i, j in zip(user_vec, game_vec):
X[i, j] = 1<prepare_x_and_y>
train_data[['SibSp', 'Survived']].groupby(['SibSp'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
Y_mat = np.zeros(( 9393, 1000))
user_vec = train_user_encoder.transform(train_users_game2_df["user_id"])
 game_vec = train_users_game2_df["predict_game_id"]
for i, j in zip(user_vec, game_vec):
Y_mat[i, j] = 1<load_from_csv>
train_data[['Parch', 'Survived']].groupby(['Parch'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
test_users = pd.read_csv(".. /input/ds2019uec-task2/test_user_ids.csv")["user_id"]<categorify>
train_data["fsize"] = train_data["SibSp"] + train_data["Parch"] + 1
test_data["fsize"] = test_data["SibSp"] + test_data["Parch"] + 1
test_user_encoder = LabelEncoder()
test_user_encoder.fit(test_users)
 print(test_user_encoder.classes_.shape )<categorify>
train_data[['fsize', 'Survived']].groupby(['fsize'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
X_test = np.zeros(( 3000, 4155))
user_vec_test = test_user_encoder.transform(test_users_game1_df["user_id"])
 game_vec = game1_encoder.transform(test_users_game1_df["game_title"])
 for i, j in zip(user_vec_test, game_vec):
X_test[i, j] = 1<define_variables>
print(train_data.Ticket.nunique())
 print(train_data.Ticket.tail() )
pred_test_mat = np.zeros(( 3000, 1000))<feature_engineering>
train_data["ticket_prefix"] = pd.Series([len(i.split())> 1 for i in train_data.Ticket], index=train_data.index )
X_test[:, X.sum(axis=0)< 100] = 0
X[:, X.sum(axis=0)< 100] = 0<train_model>
train_data[['ticket_prefix', 'Survived']].groupby(['ticket_prefix'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
for r in tqdm(range(1000)) :
if Y_mat[:, r].sum() > 100:
lr = LogisticRegression(solver='lbfgs')
 lr.fit(X, Y_mat[:, r])
 pred_test_mat[:, r] = lr.predict_proba(X_test)[:, 1]<load_from_csv>
train_data.drop("ticket_prefix", axis=1, inplace=True)
 train_data.drop("Ticket", axis=1, inplace=True)
 test_data.drop("Ticket", axis=1, inplace=True )
res_list = [""] * 3000
for i in range(3000):
res_list[i] = " ".join([str(i)for i in np.argsort(-pred_test_mat[i, :])[:10]])
 sample_submission_df = pd.read_csv(".. /input/ds2019uec-task2/sample_submission.csv")
 sub_df = sample_submission_df.copy()
sub_df["purchased_games"] = res_list<save_to_csv>
train_data.drop("Cabin", axis=1, inplace=True)
 test_data.drop("Cabin", axis=1, inplace=True )
sub_df.to_csv("logistic_regression_1000.csv", index=False )<compute_test_metric>
train_data["Embarked"] = train_data["Embarked"].fillna("S")
 train_data[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
def hamming_accuracy(prediction, true_values):

 return np.mean(np.sum(np.equal(prediction, true_values)) / float(true_values.size))
def get_score(prediction, true_values):
print("\tHamming accuracy: {:.3f}".format(hamming_accuracy(prediction, true_values)))
 print("\tAccuracy, exact matches: {:.3f}".format(accuracy_score(prediction, true_values)))
 print("\tMacro F1 Score: {:.3f}".format(f1_score(y_true=true_values, y_pred=prediction, average="macro")))
 print("\tMicro F1 Score: {:.3f}".format(f1_score(y_true=true_values, y_pred=prediction, average="micro")))
 def build_dataframe(input_data: pd.DataFrame, col_name: str, preserve_int_col_name=False)-> pd.DataFrame:

 vertices_dict = []
for i, row_i in input_data.iterrows() :
features = [int(float(x)) for x in row_i[f"{col_name}s"].split(";")]
new_v = {"id": i}
for j, f in enumerate(features):
new_v[j if preserve_int_col_name else f"{col_name}_{j}"] = f
vertices_dict += [new_v]
res_df = pd.DataFrame(vertices_dict)
 return res_df.set_index("id")
 def bool_to_int(labels: list)-> list:

 return [i for i, x in enumerate(labels)if x == 1]<load_from_csv>
train_data["Embarked"] = train_data["Embarked"].fillna("S")
 print(train_data.Embarked.isna().sum() )
graph_name = "ppi"
embeddings_path = ".. /input/embeddings.csv"
embeddings_df = pd.read_csv(embeddings_path, header=None, index_col=0)
 embeddings_df.columns = ["e_" + str(col)for col in embeddings_df.columns]
embeddings_df.head()<load_from_csv>
train_data = train_data.join(pd.get_dummies(train_data['Embarked'], prefix="Embarked_"))
test_data = test_data.join(pd.get_dummies(test_data['Embarked'], prefix="Embarked_"))
vertices_path = f".. /input/{graph_name}_train.csv"
vertices_train = pd.read_csv(vertices_path, sep=",", index_col="id")
 vertices_train["dataset"] = "train"
vertices_train.head()<load_from_csv>
train_data.drop("Embarked", axis=1, inplace=True)
 test_data.drop("Embarked", axis=1, inplace=True )
vertices_path = f".. /input/{graph_name}_test.csv"
vertices_test = pd.read_csv(vertices_path, sep=",", index_col="id")
 vertices_test["dataset"] = "test"
vertices_test.head()<prepare_output>
ss = StandardScaler()
train_y = train_data["Survived"]
train_data.drop("Survived", axis=1, inplace=True)
 scoring_method = "f1"
train_scaled = ss.fit_transform(train_data)
 test_scaled = ss.transform(test_data )
X_train_df = build_dataframe(vertices_train, "feature")
 X_train_df.head()<prepare_output>
print(train_data.isna().sum())
 print(test_data.isna().sum() )
X_test_df = build_dataframe(vertices_test, "feature")
 X_test_df.head()<prepare_output>
model = LogisticRegression(random_state=10, max_iter = 1000)
 logit_params = {
"C": [1, 3, 10, 20, 30, 40],
"solver": ["lbfgs", "liblinear"]
}
logit_gs = GridSearchCV(model, logit_params, scoring="f1", cv = 5, n_jobs=4 )
y_train_df = build_dataframe(vertices_train, "label", preserve_int_col_name=True)
 y_train_df.head()<merge>
logit_gs.fit(train_data, train_y )
X_train_df = pd.merge(X_train_df, embeddings_df, left_index=True, right_index=True, how="left")
 X_test_df = pd.merge(X_test_df, embeddings_df, left_index=True, right_index=True, how="left")
 X_train_df.head()<choose_model_class>
print(logit_gs.best_params_)
 print(logit_gs.best_score_ )
kfolds = KFold(n_splits=10)
 seed = random.randint(0, 2**32)
 sgd = SGDClassifier(loss="log", max_iter=100, tol=1e-3)
 model = OneVsRestClassifier(sgd, n_jobs=1 )<compute_train_metric>
rf_model = RandomForestClassifier()
rf_params ={
'bootstrap': [True, False],
'max_depth': [10, None],
'max_features': ['auto', 'sqrt'],
'min_samples_leaf': [1, 2, 4],
'min_samples_split': [2, 5, 10],
'n_estimators': [5, 10, 15, 20, 25, 30]}
rf_gs = GridSearchCV(rf_model, rf_params, scoring=scoring_method, cv=8, n_jobs=4 )
scores = cross_val_score(model, X_train_df, y_train_df, cv=kfolds, n_jobs=32, verbose=2, scoring="f1_micro")
 print(f"Mean crossvalidation Micro-F1: {np.mean(scores):.3f}" )<predict_on_test>
rf_gs.fit(train_data, train_y )
model.fit(X_train_df, y_train_df)
 print("Train accuracy")
 y_train_pred = model.predict(X_train_df)
 get_score(y_train_pred, y_train_df.values)
 y_test_pred = model.predict(X_test_df )<save_to_csv>
print(rf_gs.best_params_)
 print(rf_gs.best_score_ )
y_pred = [" ".join([str(y)for y in bool_to_int(x)])for x in y_test_pred]
y_pred_df = pd.DataFrame(y_pred, columns=["labels"], index=X_test_df.index)
 y_pred_df.to_csv(f"prediction_{datetime.datetime.now().strftime('%y_%m_%d_%H_%M_%S')}.csv" )<set_options>
svc_model = SVC()
test_parameters = {
"C": [1, 3, 10, 30, 100],
"kernel": ["linear", "poly", "rbf" , "sigmoid"],
}
svc_gs = GridSearchCV(svc_model, test_parameters, scoring="f1", cv=5, n_jobs=4 )
plt.rcParams['figure.figsize'] = [20, 10]
%matplotlib inline<load_from_csv>
svc_gs.fit(train_scaled, train_y )
dataset = pd.read_csv(FOLDER+'train.csv' )<choose_model_class>
print(svc_gs.best_params_)
 print(svc_gs.best_score_ )
class MeineLinearRegression(linear_model.LinearRegression):

 def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
 def R(self, y, X):
if X.shape[1] == 0:
return 0
reg = linear_model.LinearRegression()
reg.fit(X, y)
 y_pred = reg.predict(X)
 return r2_score(y_pred, y)
 def fit(self, X, y, sample_weight=None):
self = super().fit(X, y, sample_weight)
 y_pred = self.predict(X)
 Ryh = r2_score(y, y_pred)
 self.r2 = Ryh
X = np.array(X)
 y = np.array(y)
 self.rse = np.sqrt(np.sum(( self.predict(X)- y)** 2, axis=0)/ float(X.shape[0] - X.shape[1] - 1))
self.se = np.array([])
 for i, x in enumerate(X.T):
n = x - x.mean()
Rxg = self.R(x, np.delete(X, i, 1))
Sbk = np.sqrt(( 1 - Ryh)/(1 - Rxg)/(X.shape[0] - X.shape[1] - 1)) *np.std(y)/np.std(x)
 self.se = np.append(self.se, Sbk)
 self.t = self.coef_ / self.se
self.p = [2 *(1 - stats.t.cdf(np.abs(t_stu), df = y.shape[0] - X.shape[1])) for t_stu in self.t] 
<prepare_x_and_y>
lgb_model = LGBMClassifier()
test_parameters = {
"n_estimators": [int(x)for x in np.linspace(5, 30, 6)],
"reg_alpha": [0, 0.75, 1, 1.25],
"learning_rate": [0.5, 0.4, 0.35, 0.3, 0.25, 0.2],
"subsample": [0.5, 0.75, 1]
}
lgb_gs = GridSearchCV(lgb_model, test_parameters, scoring=scoring_method, cv=8, n_jobs=4 )
baseline = dataset.copy()
target = 'median_house_value'
result = []
np.seterr(all='ignore')
 for label in baseline.columns[1:-1]:
X = baseline[label]
y = baseline[target]
reg = MeineLinearRegression()
reg.fit(X.values.reshape(-1, 1), y)
 F, p = f_regression(X.values.reshape(-1, 1), y)
 result.append({'Label': label,
'Beta': '{0:.2f}'.format(reg.coef_[0]),
't':'{0:.2f}'.format(reg.t[0]),
'p-value':'{0:.6f}'.format(p[0]),
'R²':"{0:.2f}%".format(reg.r2 * 100),
'F-value':'{0:.2f}'.format(F[0])})
 pd.DataFrame(result, columns=['Label', 'Beta', 'R²', 't', 'F-value', 'p-value'] )<train_on_grid>
lgb_gs.fit(train_data, train_y )
X = baseline[baseline.columns[1:-1]]
y = baseline[target]
reg = MeineLinearRegression()
reg.fit(X, y)
 result_2 = []
for n, label in enumerate(X.columns):
result_2.append({'Label': label,
'Beta': '{0:.2f}'.format(reg.coef_[n]),
't':'{0:.2f}'.format(reg.t[n]),
'p-value':'{0:.6f}'.format(reg.p[n])})
 print("R²: {0:.2f}%".format(reg.r2 * 100))
pd.DataFrame(result_2 )<train_on_grid>
print(lgb_gs.best_params_)
 print(lgb_gs.best_score_ )
test_1 = dataset[dataset['median_house_value'] < 500000]
target = 'median_house_value'
X = test_1[test_1.columns[1:-1]]
y = test_1[target]
evaluation = linear_model.LinearRegression()
score = cross_val_score(evaluation, X, y, cv = 10, scoring='r2')
 print('R² = {0:.3f} ± {1:.3f}'.format(np.mean(score), 2*np.std(score)))
 reg = MeineLinearRegression()
reg.fit(X, y)
 result_2 = []
for n, label in enumerate(X.columns):
result_2.append({'Label': label,
'Beta': '{0:.2f}'.format(reg.coef_[n]),
't':'{0:.2f}'.format(reg.t[n]),
'p-value':'{0:.6f}'.format(reg.p[n])})
 pd.DataFrame(result_2 )<choose_model_class>
ensemble_model = VotingClassifier(estimators=[ 
("logit", logit_gs.best_estimator_), 
("rf", rf_gs.best_estimator_), 
("svc", svc_gs.best_estimator_), 
("lgb", lgb_gs.best_estimator_),
], voting = "hard" )
non_lin = dataset.copy()
stat = []
target = 'median_house_value'
for label_1 in non_lin.columns[3:-1]:
for label_2 in non_lin.columns[3:-1]:
if label_2 != label_1:
X = non_lin[label_1]/non_lin[label_2]
reg = linear_model.LinearRegression()
reg.fit(X.values.reshape(-1, 1), non_lin[target])
 F, p = f_regression(X.values.reshape(-1, 1), non_lin[target])
 y = reg.predict(X.values.reshape(-1, 1))
r2 = r2_score(non_lin[target], y)
 stat.append({'Label': label_1+'/'+label_2,
'Beta': '{0:.2f}'.format(reg.coef_[0]),
'F':'{0:.0f}'.format(F[0]),
'p-value':'{0:.6f}'.format(p[0]),
'R²':"{0:.2f}%".format(r2 * 100),
'corr':'{0:.5f}'.format(np.corrcoef(X,non_lin[target])[0][1]),
'raw_R²':r2})
 output = pd.DataFrame(stat, columns = ['Label', 'Beta', 'R²', 'p-value', 'F', 'corr', 'raw_R²'])
 output = output[output['raw_R²'] > 0.01 ]
output.sort_values(by='raw_R²', ascending=False )<feature_engineering>
ensemble_model.fit(train_data, train_y )
def add_features(dset):
mean_houses = pd.Series(dset['households']/dset['population'], name = 'mean_households')
 rooms_ratio = pd.Series(dset['total_rooms']/dset['total_bedrooms'], name = 'ratio')
 return pd.concat([mean_houses, rooms_ratio, dset], axis = 1 )<train_on_grid>
ensemble_model.score(train_data, train_y )
stat = []
target = 'median_house_value'
for n, label_1 in enumerate(non_lin.columns[3:-1]):
for label_2 in non_lin.columns[3+n:-1]:
X = non_lin[label_1]*non_lin[label_2]
reg = linear_model.LinearRegression()
reg.fit(X.values.reshape(-1, 1), non_lin[target])
 F, p = f_regression(X.values.reshape(-1, 1), non_lin[target])
 y = reg.predict(X.values.reshape(-1, 1))
r2 = r2_score(non_lin[target], y)
 stat.append({'Label': label_1+'*'+label_2,
'Beta': '{0:.5f}'.format(reg.coef_[0]),
't':'{0:.2f}'.format(np.sqrt(F[0])) ,
'p-value':'{0:.6f}'.format(p[0]),
'R²':"{0:.2f}%".format(r2 * 100),
'corr':'{0:.5f}'.format(np.corrcoef(X,non_lin[target])[0][1]),
'raw_R²':r2})
 output = pd.DataFrame(stat, columns = ['Label', 'Beta', 'R²', 'p-value', 't', 'corr', 'raw_R²'])
 output = output[output['raw_R²'] > 0.01 ]
output.sort_values(by='raw_R²', ascending=False )<load_from_csv>
preds = ensemble_model.predict(test_data )
test = pd.read_csv(FOLDER+'test.csv')
 test.head()<compute_train_metric>
output = pd.DataFrame({'PassengerId': test_data.index,
'Survived': preds})
 output.to_csv('submission.csv', index=False )