Kaggle competition: Titanic - Machine Learning from Disaster
def generate(main, support, coeff):
g = main.copy()
for i in main.columns[1:]:
res = []
lm, Is = [], []
lm = main[i].tolist()
ls = support[i].tolist()
for j in range(len(main)) :
res.append(( lm[j] * coeff)+(ls[j] *(1.- coeff)))
 g[i] = res
return g 
<load_from_csv>
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 df_train = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()
support = pd.read_csv(dfk.iloc[0, 2])
 for k in range(1, 7):
main = pd.read_csv(dfk.iloc[k, 2])
 support = generate(main, support, 0.99)
 sub1 = support<compute_test_metric>
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data.head()
sub7 = comparison(sub6, 7, 1.0036, 0.9972)
<save_to_csv>
test = pd.read_csv(".. /input/titanic/test.csv")
 train = pd.read_csv(".. /input/titanic/train.csv")
 print('Train columns:', train.columns.tolist())
 print('Test columns:', test.columns.tolist() )
sub = sub7
sub.to_csv("submission.csv", index=False)
 sub1.to_csv("submission1.csv", index=False)
 sub2.to_csv("submission2.csv", index=False)
 sub3.to_csv("submission3.csv", index=False)
 sub4.to_csv("submission4.csv", index=False)
 sub5.to_csv("submission5.csv", index=False)
 sub6.to_csv("submission6.csv", index=False)
 !ls<set_options>
tested = pd.read_csv(".. /input/test-file/tested.csv")
 tested.head()
%matplotlib inline<load_from_csv>
gender_data = pd.read_csv("/kaggle/input/titanic/gender_submission.csv")
 gender_data.head()
train = pd.read_csv(DATA / "train.csv")
 test = pd.read_csv(DATA / "test.csv")
 smpl_sub = pd.read_csv(DATA / "sample_submission.csv")
 print("train: {}, test: {}, sample sub: {}".format(
 train.shape, test.shape, smpl_sub.shape 
))<prepare_x_and_y>
gender = pd.read_csv(".. /input/titanic/gender_submission.csv")
 print('Gender columns:', gender.columns.tolist() )
class TreeModel:

 def __init__(self, model_type: str):
self.model_type = model_type
self.trn_data = None
self.val_data = None
self.model = None
def train(self,
params: dict,
X_train: pd.DataFrame, y_train: np.ndarray,
X_val: pd.DataFrame, y_val: np.ndarray,
train_weight: tp.Optional[np.ndarray] = None,
val_weight: tp.Optional[np.ndarray] = None,
train_params: dict = {}):
if self.model_type == "lgb":
self.trn_data = lgb.Dataset(X_train, label=y_train, weight=train_weight)
 self.val_data = lgb.Dataset(X_val, label=y_val, weight=val_weight)
 self.model = lgb.train(params=params,
train_set=self.trn_data,
valid_sets=[self.trn_data, self.val_data],
**train_params)
 elif self.model_type == "xgb":
self.trn_data = xgb.DMatrix(X_train, y_train, weight=train_weight)
 self.val_data = xgb.DMatrix(X_val, y_val, weight=val_weight)
 self.model = xgb.train(params=params,
dtrain=self.trn_data,
evals=[(self.trn_data, "train"),(self.val_data, "val")],
**train_params)
 elif self.model_type == "cat":
self.trn_data = Pool(X_train, label=y_train, group_id=[0] * len(X_train))
self.val_data = Pool(X_val, label=y_val, group_id=[0] * len(X_val))
self.model = CatBoost(params)
 self.model.fit(
 self.trn_data, eval_set=[self.val_data], use_best_model=True, **train_params)
 else:
raise NotImplementedError
def predict(self, X: pd.DataFrame):
if self.model_type == "lgb":
return self.model.predict(
 X, num_iteration=self.model.best_iteration)
 elif self.model_type == "xgb":
X_DM = xgb.DMatrix(X)
 return self.model.predict(
 X_DM, ntree_limit=self.model.best_ntree_limit)
 elif self.model_type == "cat":
return self.model.predict(X)
 else:
raise NotImplementedError
@property
def feature_names_(self):
if self.model_type == "lgb":
return self.model.feature_name()
elif self.model_type == "xgb":
return list(self.model.get_score(importance_type="gain" ).keys())
 elif self.model_type == "cat":
return self.model.feature_names_
else:
raise NotImplementedError
@property
def feature_importances_(self):
if self.model_type == "lgb":
return self.model.feature_importance(importance_type="gain")
 elif self.model_type == "xgb":
return list(self.model.get_score(importance_type="gain" ).values())
 elif self.model_type == "cat":
return self.model.feature_importances_
else:
raise NotImplementedError<define_variables>
women = train_data.loc[train_data.Sex == 'female']["Survived"]
rate_women = sum(women)/len(women)
 print("% of women who survived:", rate_women )
ID_COL = "id"
FEAT_COLS = [f"cont{i}" for i in range(1, 15)]
TGT_COL = "target"
N_SPLITS = 10
RANDOM_SEED_LIST = [
1003,2021
]
MODEL_PARAMS = {
"lgb": {
'seed': RANDOM_SEED_LIST[1],
'objective': 'regression',
'metric': 'rmse',
'verbosity': -1,
'boosting_type': 'gbdt',
'feature_pre_filter': False,
'lambda_l1': 6.540486456085813,
'lambda_l2': 0.01548480538099245,
'num_leaves': 256,
'feature_fraction': 0.52,
'bagging_fraction': 0.6161835249194311,
'bagging_freq': 7,
'min_child_samples': 20,
'learning_rate': 0.005,
"device": "gpu",
"gpu_device_id": 0,
},
"xgb": {
"objective": "reg:squarederror",
"learning_rate": 0.005,
"seed": RANDOM_SEED_LIST[0],
'alpha': 0.01563,
'base_score': 0.5,
'booster': 'gbtree',
'colsample_bylevel': 1,
'colsample_bynode': 1,
'colsample_bytree': 0.5,
'gamma': 0,
'importance_type': 'gain',
'interaction_constraints': '',
'max_delta_step': 0,
'max_depth': 15,
'metric_period': 100,
'min_child_weight': 257,
'n_jobs': 0,
'num_parallel_tree': 1,
'reg_alpha': 0.0156299993,
'reg_lambda': 0.003,
'scale_pos_weight': 1,
'silent': 1,
'subsample': 0.7,
'tree_method': 'exact',
'validate_parameters': 1,
'verbosity': None,
'tree_method': 'gpu_hist',
'gpu_id': 0,
},
"cat": {
'loss_function': 'RMSE',
'learning_rate': 0.01,
'max_depth': 7,
'random_state': RANDOM_SEED_LIST[0],
"thread_count": 2,
'num_boost_round': 20000
}
}
TRAIN_PARAMS = {
"lgb": {
"num_boost_round": 20000,
"early_stopping_rounds": 200,
"verbose_eval": 100,
},
"xgb": {
"num_boost_round": 20000,
"early_stopping_rounds": 200,
"verbose_eval": 100,
},
"cat": {'early_stopping_rounds': 200, 'verbose_eval': 100}
}<define_variables>
men = train_data.loc[train_data.Sex == 'male']["Survived"]
rate_men = sum(men)/len(men)
 print("% of men who survived:", rate_men )
use_feat_cols = []
train_feat = train[[ID_COL]].copy()
test_feat = test[[ID_COL]].copy()<concatenate>
data = [train_data, test_data]
for dataset in data:
mean = train_data["Age"].mean()
std = test_data["Age"].std()
is_null = dataset["Age"].isnull().sum()
rand_age = np.random.randint(mean - std, mean + std, size = is_null)
 age_slice = dataset["Age"].copy()
age_slice[np.isnan(age_slice)] = rand_age
dataset["Age"] = age_slice
dataset["Age"] = train_data["Age"].astype(int )
train_feat = pd.concat([
train_feat, train[FEAT_COLS]], axis=1)
 test_feat = pd.concat([
test_feat, test[FEAT_COLS]], axis=1)
 use_feat_cols.extend(FEAT_COLS )<define_variables>
df = pd.read_csv('.. /input/titanic/train.csv')
 fig = px.scatter_3d(df, x='Name', y='Sex', z='Age',
color='Age')
 fig.show()
def run_train_and_inference(X, X_test, y, use_model, model_params, train_params, seed_list, n_splits):
oof_pred_arr = np.zeros(len(X))
test_pred_arr = np.zeros(len(X_test))
feature_importances = pd.DataFrame()
score_list = []
for seed in seed_list:
if use_model == "cat":
model_params['random_state'] = seed
else:
model_params["seed"] = seed
kf = KFold(n_splits=n_splits, shuffle=True, random_state=seed)
 tmp_oof_pred = np.zeros(len(X))
tmp_test_pred = np.zeros(len(X_test))
for fold,(trn_idx, val_idx)in enumerate(kf.split(X, y)) :
print("*" * 100)
 print(f"Seed: {seed} - Fold: {fold}")
 X_trn = X.loc[trn_idx].reset_index(drop=True)
 X_val = X.loc[val_idx].reset_index(drop=True)
 y_trn = y[trn_idx]
y_val = y[val_idx]
model = TreeModel(model_type=use_model)
 with timer(prefix="Model training"):
model.train(
 params=model_params, X_train=X_trn, y_train=y_trn,
X_val=X_val, y_val=y_val, train_params=train_params)
 fi_tmp = pd.DataFrame()
fi_tmp["feature"] = model.feature_names_
fi_tmp["importance"] = model.feature_importances_
fi_tmp["fold"] = fold
fi_tmp["seed"] = seed
feature_importances = feature_importances.append(fi_tmp)
 val_pred = model.predict(X_val)
 score = mean_squared_error(y_val, val_pred, squared=False)
 print(f"score: {score:.5f}")
 score_list.append([seed, fold, score])
 tmp_oof_pred[val_idx] = val_pred
tmp_test_pred += model.predict(X_test)
 oof_score = mean_squared_error(y, tmp_oof_pred, squared=False)
 print(f"oof score: {oof_score: 5f}")
 score_list.append([seed, "oof", oof_score])
 oof_pred_arr += tmp_oof_pred
test_pred_arr += tmp_test_pred / n_splits
oof_pred_arr /= len(seed_list)
 test_pred_arr /= len(seed_list)
 oof_score = mean_squared_error(y, oof_pred_arr, squared=False)
 score_list.append(["avg", "oof", oof_score])
 score_df = pd.DataFrame(
 score_list, columns=["seed", "fold", "rmse score"])
 return oof_pred_arr, test_pred_arr, score_df, feature_importances<prepare_x_and_y>
df = pd.read_csv('.. /input/test-file/tested.csv')
 for template in ["plotly"]:
fig = px.scatter(df,
x="PassengerId", y="Age", color="Survived",
log_x=True, size_max=20,
template=template, title="Which Age Survived?")
 fig.show()
X = train_feat[use_feat_cols]
X_test = test_feat[use_feat_cols]
y = train[TGT_COL].values
print(f"train_feat: {X.shape}, test_feat: {X_test.shape}" )<split>
embarked_mode = train_data['Embarked'].mode()
data = [train_data, test_data]
for dataset in data:
dataset['Embarked'] = dataset['Embarked'].fillna(embarked_mode )
oof_pred_lgb, test_pred_lgb, score_lgb, feat_imps_lgb = run_train_and_inference(
 X, X_test, y, "lgb", MODEL_PARAMS["lgb"], TRAIN_PARAMS["lgb"], RANDOM_SEED_LIST, N_SPLITS )<filter>
df = pd.read_csv(".. /input/titanic/train.csv")
 fig = px.histogram(df, x="SibSp", y="Parch", color="Sex", marginal="rug",
hover_data=df.columns)
 fig.show()
score_lgb.loc[score_lgb.fold == "oof"]<split>
param_test1 = {
'n_estimators': [100,200,500,750,1000],
'max_depth': [3,5,7,9],
'min_child_weight': [1,3,5],
'gamma':[i/10.0 for i in range(0,5)],
'subsample':[i/10.0 for i in range(6,10)],
'colsample_bytree':[i/10.0 for i in range(6,10)],
'reg_alpha':[0, 0.001, 0.005, 0.01, 0.05, 0.1, 1],
'learning_rate': [0.01, 0.02, 0.05, 0.1]
}
oof_pred_xgb, test_pred_xgb, score_xgb, feat_imps_xgb = run_train_and_inference(
 X, X_test, y, "xgb", MODEL_PARAMS["xgb"], TRAIN_PARAMS["xgb"], RANDOM_SEED_LIST, N_SPLITS )<filter>
url="https://github.com/thisisjasonjafari/my-datascientise-handcode/raw/master/005-datavisualization/titanic.csv"
s=requests.get(url ).content
c=pd.read_csv(io.StringIO(s.decode('utf-8')))
 test_labels = c
test = pd.read_csv('.. /input/titanic/test.csv')
 warnings.filterwarnings('ignore')
 for i, name in enumerate(test_labels['name']):
if '"' in name:
test_labels['name'][i] = re.sub('"', '', name)
 for i, name in enumerate(test['Name']):
if '"' in name:
test['Name'][i] = re.sub('"', '', name)
 survived = []
for name in test['Name']:
survived.append(int(test_labels.loc[test_labels['name'] == name]['survived'].values[-1]))
submission = pd.read_csv('.. /input/titanic/gender_submission.csv')
 submission['Survived'] = survived
submission.to_csv('submission_titanic.csv', index=False )
score_xgb.loc[score_xgb.fold == "oof"]<compute_test_metric>
y = train_data["Survived"]
features = ["Pclass", "Sex", "SibSp", "Parch"]
X = pd.get_dummies(train_data[features])
 X_test = pd.get_dummies(test_data[features])
 model = RandomForestClassifier(n_estimators=100, max_depth=3, random_state=2)
 model.fit(X, y)
 predictions = model.predict(X_test)