Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<groupby>
sns.set(style="darkgrid")
 SEED = 31
random.seed(SEED)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
test = online_sales.groupby(['Date'])['Quantity'].sum().reset_index()
test.head()<groupby>
data = pd.read_csv("/kaggle/input/titanic/train.csv")
 data_test = pd.read_csv("/kaggle/input/titanic/test.csv" )
test = online_sales.groupby(['Date', 'Product SKU'])['Quantity'].sum().reset_index()
test.head()<groupby>
y = data["Survived"]
x = data.copy()
x_test = data_test.copy()
x_full = pd.concat([x, x_test])
 n = len(x)
 n_test = len(x_test)
 n_full = n + n_test
test = online_sales.groupby(['Date'] ).agg({'Quantity': 'sum',
'Revenue': 'sum',
'Tax': 'sum',
'Product SKU': 'count',
'Transaction ID': 'count',
} ).reset_index()
test.head()<sort_values>
( x_full.drop(["Survived"], axis = 1 ).isna().sum() / n_full)* 100
online_sales.sort_values(by=['Quantity'], ascending = False ).head(15 )<drop_column>
x_full[x_full["Fare"].isnull() ]
test.rename(index=str, columns={"Quantity": "Total Quantity", "Revenue": "Total Revenue"}, inplace = True)
 test.drop(columns=['Product SKU', 'Transaction ID'], inplace = True)
 test.head()<load_from_csv>
x_full[x_full["Embarked"].isnull() ]
online_sales = online_sales = pd.read_csv('/kaggle/input/uisummerschool/Online_sales.csv')
 backup = online_sales.copy()
daily_online_revenue = online_sales.groupby(['Date'])['Revenue'].sum().reset_index()
daily_online_revenue.tail()<create_dataframe>
embarked_survived = x.groupby(["Embarked", "Survived"])["PassengerId"].count()
embarked_survived
add_data = [['2017-12-01', 0], ['2017-12-02', 0], ['2017-12-03', 0],
['2017-12-04', 0], ['2017-12-05', 0], ['2017-12-06', 0],
['2017-12-07', 0], ['2017-12-08', 0], ['2017-12-09', 0],
['2017-12-10', 0], ['2017-12-11', 0], ['2017-12-10', 0],
['2017-12-13', 0], ['2017-12-14', 0]
]
add_data_df = pd.DataFrame(add_data, columns = ['Date', 'Revenue'])
 add_data_df['Date'] = add_data_df['Date'].astype(str)
 add_data_df['Date'] = pd.to_datetime(add_data_df['Date'])
 daily_online_revenue = daily_online_revenue.append(add_data_df)
 df_rev = daily_online_revenue.copy()
daily_online_revenue.tail(20 )<prepare_output>
def display_survival_prob(data, feature):
feature_survived = data.groupby([feature, "Survived"])["PassengerId"].count()
classes = set(feature_survived.index.get_level_values(0))
surv_counts = {key: {} for key in classes}
for c in classes:
for survive in range(0, 2):
surv_counts[c][survive] = 0
if survive in feature_survived[c]:
surv_counts[c][survive] = feature_survived[c][survive]
df_surv = pd.DataFrame(surv_counts)
 surv_percentages = {}
for col in df_surv.columns:
surv_percentages[col] = [(count / df_surv[col].sum())* 100 for count in df_surv[col]]
df_survived_percentages = pd.DataFrame(surv_percentages ).transpose()
class_names = tuple(classes)
 bar_count = np.arange(len(class_names))
bar_width = 0.5
not_survived = df_survived_percentages[0]
survived = df_survived_percentages[1]
plt.figure(figsize=(10, 6))
plt.bar(bar_count, not_survived, color="
plt.bar(bar_count, survived, bottom=not_survived, color="
plt.xlabel("{}".format(feature), size=15, labelpad=20)
 plt.ylabel("Survival Percentage", size=15, labelpad=20)
 plt.xticks(bar_count, class_names)
 plt.tick_params(axis="x", labelsize=15)
 plt.tick_params(axis="y", labelsize=15)
 plt.legend(loc="upper left", bbox_to_anchor=(1, 1), prop={"size": 15})
 plt.title("Survival Percentage by {}".format(feature), size=18, y=1.05)
 plt.show()
daily_online_revenue = preprocess(df_rev ).set_index('Date')
 daily_online_revenue.head()<split>
embarked_class_passengers = x.groupby("Pclass")["Embarked"].value_counts()
print(embarked_class_passengers )
def split_train_test(dataset, end_of_training_date):
training_data = daily_online_revenue.loc['2017-01-05':end_of_training_date]
testing_data = daily_online_revenue["2017-12-01":]
return training_data, testing_data<prepare_x_and_y>
x2 = x.copy()
x2["Cabin"] = x2["Cabin"].map(lambda cabin: cabin if pd.isnull(cabin)else cabin[0])
 x2["Cabin"].sample(10 )
def split_label_and_predictor(train_or_test_data):
y_data = train_or_test_data['today_revenue']
x_data = train_or_test_data[['d-1_rev','d-2_rev','d-3_rev','d-4_rev','d-5_rev','d-6_rev','d-7_rev','d-8_rev','d-9_rev','d-10_rev']]
return x_data, y_data<split>
x2.groupby(["Cabin", "Pclass"])["PassengerId"].count()
end_of_training_date = "2017-11-30"
training_data, test_data = split_train_test(daily_online_revenue,end_of_training_date)
<split>
class_cabin_passengers = x2.groupby("Pclass")["Cabin"].value_counts()
print("Number of passengers without a cabin registered: {}".format(sum(x2["Cabin"].isnull())))
 print("Number of passengers without a cabin registered that survived: {}".format(sum(( x2["Survived"] == 1)&(x2["Cabin"].isnull()))))
print("Number of passengers without a cabin registered that didn't survived: {}".format(sum(( x2["Survived"] == 0)&(x2["Cabin"].isnull()))))
print("Total number of passengers that didn't survived: {}".format(sum(x["Survived"] == 0)))
 print("=================")
 print("Passengers from class 1 with cabin registered: {}".format(class_cabin_passengers[1].sum()))
print("Total number of Passengers with cabin registered: {}".format(( ~x2["Cabin"].isnull() ).sum()))
print("Number of passengers with a cabin registered that survived: {}".format(sum(( x2["Survived"] == 1)& ~(x2["Cabin"].isnull()))))
print("Total number of Passengers of class 1: {}".format(( x2["Pclass"] == 1 ).sum()))
x_train, y_train = split_label_and_predictor(training_data)
 x_test, y_test = split_label_and_predictor(test_data)
<train_model>
class MissingValuesTransformer(BaseEstimator, TransformerMixin):
def __init__(self):
print("In the MissingValuesTransformer init method")
 def fit(self, x, y = None):
self._median_gender_class_age = x.groupby(["Sex", "Pclass"])["Age"].median()
self._median_class_fare = x.groupby(["Pclass"])["Fare"].median()
self._count_class_embarked = x.groupby("Pclass")["Embarked"].value_counts()
return self
def transform(self, x, y = None):
x["Cabin"] = x["Cabin"].map(lambda cabin: cabin if pd.isnull(cabin)else cabin[0])
 self._count_class_cabin = x.groupby("Pclass")["Cabin"].value_counts()
x = x.apply(self._fill_na_age, axis = 1)
 x = x.apply(self._fill_na_fare, axis = 1)
 x = x.apply(self._fill_na_embarked, axis = 1)
 x = x.apply(self._fill_na_cabin, axis = 1)
 return x
def _fill_na_age(self, row):
row["Age"] = self._median_gender_class_age[row["Sex"]][row["Pclass"]] if pd.isnull(row["Age"])else row["Age"]
return row
def _fill_na_fare(self, row):
row["Fare"] = self._median_class_fare[row["Pclass"]] if pd.isnull(row["Fare"])else row["Fare"]
return row
def _fill_na_embarked(self, row):
row_class = row["Pclass"]
total_class_passengers = self._count_class_embarked[row_class].sum()
class_embarked_dist = self._count_class_embarked[row_class] / total_class_passengers
if pd.isnull(row["Embarked"]):
row["Embarked"] = random.choices(population = class_embarked_dist.index,
weights = class_embarked_dist,
k = 1)[0]
return row
def _fill_na_cabin(self, row):
row_class = row["Pclass"]
total_class_passengers = self._count_class_cabin[row_class].sum()
class_cabin_dist = self._count_class_cabin[row_class] / total_class_passengers
if pd.isnull(row["Cabin"]):
row["Cabin"] = random.choices(population = class_cabin_dist.index,
weights = class_cabin_dist,
k = 1)[0]
return row
def fit(x_train, y_train):
model = RandomForestRegressor(random_state=1)
 model.fit(x_train, y_train)
 return model
def predict(model, x_test):
y_pred = model.predict(x_test)
 return y_pred
model = fit(x_train, y_train)
<predict_on_test>
pipeline = Pipeline(steps=[ 
("preprocessor", MissingValuesTransformer())
 ])
 filled_x = pipeline.fit_transform(x)
 filled_x.sample(10 )
df_rev2 = df_rev.copy()
n_iteration = len(x_test)
 result = []
for i in range(n_iteration):
y_pred = predict(model, pd.DataFrame(x_test.iloc[i] ).transpose())
 result.append(y_pred[0])
 df_rev2.loc[df_rev2["Date"]==x_test.index[i],"Revenue"] = y_pred
daily_online_revenue = preprocess(df_rev2 ).set_index('Date')
 _, testing_data = split_train_test(daily_online_revenue,end_of_training_date)
 x_test, _ = split_label_and_predictor(testing_data)
 result<compute_test_metric>
x2["FamilySize"] = x2["SibSp"] + x2["Parch"] + 1
x2.drop(["SibSp", "Parch"], axis = 1, inplace = True)
 x2.sample(10 )
comparison = pd.DataFrame({"Prediction":result,"Actual":y_test})
 comparison.index = y_test.index
error = sqrt(mean_squared_error(comparison["Actual"], comparison["Prediction"]))
print("Error Score(RMSE)= {}".format(round(error,2)))
 historical = pd.DataFrame(y_train ).rename(columns={"Revenue":"Actual"} ).tail(14)
 pd.concat([historical,comparison],sort=True ).plot() ;<create_dataframe>
def get_family_group(size):
if size == 1:
return "Alone"
elif size <= 4:
return "Small"
elif size <= 6:
return "Medium"
else:
return "Large"
x2["FamilySize"] = x2["FamilySize"].map(lambda s: get_family_group(s))
x2.sample(10 )
formatted_result = pd.DataFrame(result ).reset_index().rename(columns={"index":"Id",0:"Revenue"})
 display(formatted_result)
 formatted_result.to_csv("result.csv",index=False)
<install_modules>
x2["NameTitle"] = x2["Name"].map(lambda name: name.split(", ")[1].split(".")[0])
 x2.sample(5 )
!pip install catboost==0.13.1<set_options>
def get_reduced_title(title):
return title if title in ["Mr", "Mrs", "Miss", "Master"] else "Other"
x2["NameTitle"] = x2["NameTitle"].map(lambda title: get_reduced_title(title))
x2.sample(5 )
warnings.filterwarnings("ignore", category=DeprecationWarning)
 warnings.filterwarnings("ignore", category=FutureWarning)
 warnings.filterwarnings("ignore", category=UserWarning )<import_modules>
pds = pd.Series([3.3, 40.5, 23.78, 56.4, 15,2])
 pds.isin(x2["Fare"] )
import gc
import numpy as np
import pandas as pd
from time import time
from sklearn.metrics import roc_auc_score, classification_report
from catboost import Pool, CatBoost, CatBoostClassifier, MetricVisualizer<load_from_csv>
pipeline = Pipeline(steps=[ 
("preprocessor", MissingValuesTransformer()), 
("featureEngineering", FeatureEngineeringTransformer())
 ])
 output = pipeline.fit_transform(x)
 output.head(7 )
basepath = ".. /input/"
df_train = pd.read_csv(f"{basepath}train/train.csv", index_col=0, low_memory=False)
 df_test = pd.read_csv(f"{basepath}test/test.csv", index_col=0, low_memory=False)
 group_col = 'group_id'
target = 'target'
df_train.shape, df_test.shape<drop_column>
categorical_columns = ["Pclass", "Sex", "Cabin", "Embarked", "FamilySize", "NameTitle"]
feature_transformer = ColumnTransformer(
 transformers=[ 
('age', SimpleImputer() , ["Age"]), 
('fare', OrdinalEncoder() , ["Fare"]), 
('categorical', OneHotEncoder() , categorical_columns)
 ]
)
df_train = df_train.drop(df_train.index[[0,1,2,3]])
 df_train = df_train.drop(['feature_13', 'feature_32'], axis=1)
 df_test = df_test.drop(['feature_13', 'feature_32'], axis=1)
 df_train = df_train.rename(columns={'feature_18': 'pos'})
 df_test = df_test.rename(columns={'feature_18': 'pos'} )<data_type_conversions>
pipeline = Pipeline(steps=[ 
("preprocessor", MissingValuesTransformer()), 
("featureEngineering", FeatureEngineeringTransformer()), 
("columnTransformer", feature_transformer)
 ])
 output = pipeline.fit_transform(x, y)
 print(output.todense() [0:9,:] )
num_features = list(df_train.select_dtypes(include=[np.float64, np.int64] ).columns)
 num_features.remove(target)
 num_features.remove(group_col)
 mins = df_train[num_features].min()
maxs = df_train[num_features].max()
difs = np.abs(maxs - mins)
 vals = mins - 5 * difs
for col in num_features:
df_train[col] = df_train[col].fillna(vals[col])
 df_test[col] = df_test[col].fillna(vals[col])
 df_train[num_features] = df_train[num_features].astype(np.float32)
 df_test[num_features] = df_test[num_features].astype(np.float32)
 f"{gc.collect() } objects collected"<categorify>
preprocessing_pipeline = Pipeline(steps=[ 
("preprocessor", MissingValuesTransformer()), 
("featureEngineering", FeatureEngineeringTransformer()), 
("columnTransformer", feature_transformer)
 ] )
def add_aggregates(df, col):
df[f"{col}_min"] = df.groupby(group_col)[col].transform(np.min)
 df[f"{col}_avg"] = df.groupby(group_col)[col].transform(np.mean)
 df[f"{col}_q50"] = df.groupby(group_col)[col].transform(np.median)
 df[f"{col}_max"] = df.groupby(group_col)[col].transform(np.max)
 df[f"{col}_var"] = df.groupby(group_col)[col].transform(np.var)
 df[f"{col}_sum"] = df.groupby(group_col)[col].transform(np.sum)
 df[f"{col}_cmp_min"] = df[col] == df[f"{col}_min"]
df[f"{col}_cmp_max"] = df[col] == df[f"{col}_max"]
df[f"{col}_cmp_avg"] = df[col] <= df[f"{col}_avg"]
df[f"{col}_cmp_q50"] = df[col] <= df[f"{col}_q50"]
df[f"{col}_scaled_min"] = df[col] / df[f"{col}_min"]
df[f"{col}_scaled_avg"] = df[col] / df[f"{col}_avg"]
df[f"{col}_scaled_q50"] = df[col] / df[f"{col}_q50"]
df[f"{col}_scaled_max"] = df[col] / df[f"{col}_max"]
df[f"{col}_scaled_sum"] = df[col] / df[f"{col}_sum"]<data_type_conversions>
x_clean = preprocessing_pipeline.fit_transform(x)
 x_test_clean = preprocessing_pipeline.transform(x_test )
cat_features = list(df_train.select_dtypes(include=[np.object] ).columns)
 df_train[cat_features] = df_train[cat_features].fillna('N/A')
 df_test[cat_features] = df_test[cat_features].fillna('N/A')
 all_data = pd.concat(( df_train, df_test))
for col in cat_features:
cats = all_data[col].unique()
df_train[col] = df_train[col].astype('category', categories = cats)
 df_test[col] = df_test[col].astype('category', categories = cats)
 for col in cat_features:
df_train[col] = df_train[col].cat.codes + 1
df_test[col] = df_test[col].cat.codes + 1
del all_data
f"{gc.collect() } objects collected"<define_variables>
x_train, x_valid, y_train, y_valid = train_test_split(x_clean, y, train_size=0.8, test_size=0.2, random_state=0 )
features = num_features + cat_features
f"Total {len(features)} features: {len(num_features)} num., {len(cat_features)} cat."<compute_train_metric>
model = RandomForestClassifier(criterion='gini',
n_estimators=1100,
max_depth=5,
min_samples_split=4,
min_samples_leaf=5,
max_features='auto',
oob_score=True,
random_state=42,
n_jobs=-1,
verbose=1 )
def score_model(model, pool):
ys = pool.get_label()
ys_pred = model.predict(pool)
 results = {'auc': roc_auc_score(ys, ys_pred)}
results.update(classification_report(ys, ys_pred, output_dict=True))
return results
def print_results(r):
if 'time' in r:
print(f"Train time: {r['time'][0]:2d}m {r['time'][1]:2d}s")
 print(f"AUC score: {r['auc']:1.5f}")
 print(f"Class Neg:")
 print(f" precision: {r['0']['precision']:1.5f}")
 print(f" recall: {r['0']['recall']:1.5f}")
 print(f"Class Pos:")
 print(f" precision: {r['1']['precision']:1.5f}")
 print(f" recall: {r['1']['recall']:1.5f}" )<categorify>
modeling_pipeline = Pipeline(steps=[ 
("model", model)
 ] )
def make_pool(df, labeled=True):
return Pool(
 data = df[features],
label = df[target] if labeled else None,
cat_features = cat_features,
group_id = df[group_col], 
)
 def space_split(df, val_size=0.1):
df.sort_values([group_col, 'pos'])
 cutoff = df[group_col].iloc[int(( 1 - val_size)* len(df)) ]
pt1, pt2 = df[df[group_col] < cutoff], df[df[group_col] >= cutoff]
return make_pool(pt1), make_pool(pt2 )<train_model>
modeling_pipeline.fit(x_train, y_train )
def full_train(model, train_data, test_data, name):
train_pool = make_pool(train_data)
 t_start = time()
model.fit(train_pool)
 t_elapsed = int(time() - t_start)
 m, s = t_elapsed // 60, t_elapsed % 60
print(f"Train time: {m}m {s}s")
 if target in test_data.columns:
test_pool = make_pool(test_data)
 return score_model(model, test_pool)
 else:
id_test = test_data.index
xs_test = make_pool(test_data, labeled=False)
 ys_prob = model.predict_proba(xs_test)
 yP_prob = [y2 for y1,y2 in ys_prob]
answer = pd.DataFrame({'Id': id_test, 'target': yP_prob})
 answer.to_csv(f'{name}.csv', sep=',', index=False )<train_model>
preds = modeling_pipeline.predict(x_valid)
 preds
def validate(model, data, val_size=0.1, rounds=100, fit=True):
results = {}
learn, val = space_split(data, val_size)
 if fit:
t_start = time()
model.fit(
 X=learn,
eval_set=val,
use_best_model=True,
early_stopping_rounds=rounds,
verbose=False,
plot=True 
)
 t_elapsed = int(time() - t_start)
 results['time'] = t_elapsed // 60, t_elapsed % 60
results.update(score_model(model, val))
return results<init_hyperparams>
accuracy_score(y_valid, preds )
common = {
'loss_function': 'CrossEntropy',
'eval_metric': 'F1',
'od_type': 'IncToDec',
'od_pval': 1e-5,
'boosting_type': 'Ordered',
'bootstrap_type': 'Bernoulli',
'one_hot_max_size': 3,
'random_seed': 51,
'task_type': 'GPU',
}<init_hyperparams>
preds_test = modeling_pipeline.predict(x_test_clean )
params = {
**common, 'train_dir': 'p08-2naf',
'iterations': 1500,
'learning_rate': 0.05,
'depth': 8,
'l2_leaf_reg': 12.0,
'random_strength': 3.0,
'subsample': 0.5,
}<train_model>
output = pd.DataFrame({"PassengerId": x_test["PassengerId"],
"Survived": preds_test})
 output.head()
model = CatBoostClassifier(**params)
 full_train(model, df_train, df_test, 'm13-kernel-catboost' )<import_modules>
output.to_csv('submission.csv', index=False )