Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<find_best_params>
import numpy as np
import pandas as pd
for epoch in range(6):
losses = []
for i,(batch_data, batch_label)in enumerate(train_dataloader):
batch_data, batch_label = Variable(batch_data), Variable(batch_label)
 if torch.cuda.is_available() :
batch_data = batch_data.to('cuda')
 batch_label = batch_label.to('cuda')
 log_probs = model(batch_data)
 loss = loss_function(log_probs, batch_label.transpose(0, -1))
model.zero_grad()
loss.backward()
optimizer.step()
losses.append(loss.item())
 corrects = log_probs.data.cpu().numpy().argmax(axis=1)== batch_label.data.cpu().numpy()
train_accuracy = corrects.astype(np.long ).mean()
val_losses = []
val_accuracies = []
for test_data, test_label in dev_dataloader:
test_data, test_label = Variable(test_data), Variable(test_label)
 if torch.cuda.is_available() :
test_data = test_data.to('cuda')
 test_label = test_label.to('cuda')
 val_log_probs = model(test_data)
 val_loss = loss_function(val_log_probs, test_label)
 val_corrects = val_log_probs.data.cpu().numpy().argmax(axis=1)== test_label.data.cpu().numpy()
val_accuracy = val_corrects.astype(np.float64 ).mean()
val_losses.append(val_loss.item())
 val_accuracies.append(val_accuracy)
 message = "Epoch: {epoch:<5d} Iteration: {iteration:<5d} Loss: {loss:<.3} Val Loss: {val_loss:<.3} Train Accuracy: {train_acc:<.3} Test Accuracy: {test_acc:<.3}".format(
 epoch=epoch,
iteration= i,
loss=sum(losses)/len(losses),
val_loss=sum(val_losses)/len(val_losses),
train_acc=train_accuracy,
test_acc=sum(val_accuracies)/len(val_accuracies)
)
 print(message )<load_from_csv>
data_train = pd.read_csv('.. /input/train.csv')
 data_train.head()
test_df = pd.read_csv('/kaggle/input/koreangenderbiasdetection/test.gender_bias.no_label.csv' )<define_variables>
data_test = pd.read_csv('.. /input/test.csv')
 data_test.head()
test_sentences = [list(d)for d in test_df['comments'].to_list() ]<create_dataframe>
df = data_train.append(data_test, sort = True)
 df.shape
test_sentences_processed = process_sentences(test_sentences, word2idx, sentence_length=SENTENCE_LENGTH)
 test_mask = make_mask(test_sentences, sentence_length=SENTENCE_LENGTH)
 test_dataset = DatasetLoader(test_sentences_processed, [0 for i in range(len(test_sentences_processed)) ])
 test_dataloader = DataLoader(test_dataset, batch_size=1 )<feature_engineering>
%matplotlib inline
plt.style.use('bmh')
 plt.rc('font', family='DejaVu Sans', size=13 )
test_df['label'] = res<count_values>
cat_list = ['Cabin', 'Embarked', 'Name', 'Parch', 'Pclass', 'Sex', 'SibSp', 'Ticket']
for n, i in enumerate(cat_list):
cat_num = len(df[i].value_counts().index)
 print('The feature "%s" has %d values.' %(i, cat_num))
test_df['label'].value_counts()<feature_engineering>
df.isnull().sum()
test_df['label'] = test_df['label'].map(lambda x: True if x else False )<save_to_csv>
df[df.Fare.isnull() ]
test_df.to_csv('/kaggle/working/prediction.csv', index=None, header=True )<set_options>
df[(df.Pclass == 3)&(df.Age > 60)&(df.Sex == 'male')]
sns.set()
%matplotlib inline
warnings.filterwarnings('ignore')
 %load_ext tensorboard.notebook
pd.set_option('display.max_columns', None)
<load_from_csv>
fare_mean = df[(df.Pclass == 3)&(df.Age > 60)&(df.Sex == 'male')].Fare.mean()
df.loc[df.PassengerId == 1044, 'Fare'] = fare_mean
df[df.PassengerId == 1044]
odf=pd.read_csv('/kaggle/input/equipfails/equip_failures_training_set.csv',index_col=0)
 odft=pd.read_csv('/kaggle/input/equipfails/equip_failures_test_set.csv',index_col=0 )<data_type_conversions>
df[df.Embarked.isnull() ]
df=odf.replace({'na':-999999})
 Xt=odft.replace({'na':-999999})
 Xt=Xt.astype(float)
 df=df.astype(float)
 df['target']=df['target'].astype(int )<prepare_x_and_y>
df.Embarked = df.Embarked.fillna('C' )
X=df.iloc[:,1:]
y=df.iloc[:,0]
Xn=normalize(X)
 Xtn=normalize(Xt )<split>
df.Cabin = df.Cabin.fillna('0')
 len(df.Cabin.value_counts().index )
X_train, X_test, y_train, y_test = train_test_split(Xn, y, test_size=0.2, random_state=8 )<compute_train_metric>
df['CabinCat'] = pd.Categorical(df.Cabin.apply(lambda x : x[0])).codes
my_class = ExtraTreesClassifier(random_state=0)
 my_class.fit(X_train, y_train)
 y_pred= clf.predict(X_test)
 print('accuracy: {}'.format(accuracy_score(y_test, y_pred)))
 print(f'F1: {f1_score(y_test,y_pred)}')
 confusion_matrix(y_test,y_pred )<compute_test_metric>
import re
from sklearn.preprocessing import LabelEncoder
my_class = AdaBoostClassifier(random_state=0)
 my_class.fit(X_train, y_train)
 y_pred= my_class.predict(X_test)
 print('accuracy: {}'.format(accuracy_score(y_test, y_pred)))
 print(f'F1: {f1_score(y_test,y_pred)}')
 confusion_matrix(y_test,y_pred )<compute_train_metric>
df['Title'] = df.Name.apply(lambda x : re.search('([a-zA-Z]+)\.', x ).group(1))
title_mapping = {'Mr' : 1, 'Miss' : 2, 'Mrs' : 3, 'Master' : 4, 'Dr' : 5, 'Rev' : 6, 'Major' : 7, 'Col' : 7, 'Mlle' : 2, 'Mme' : 3, 'Don' : 9, 'Dona' : 9, 'Lady' : 10, 'Countess' : 10, 'Jonkheer' : 10, 'Sir' : 9, 'Capt' : 7, 'Ms' : 2}
df['TitleCat'] = df.Title.map(title_mapping)
 df['FamilySize'] = df.SibSp + df.Parch + 1
df['FamilyName'] = df.Name.apply(lambda x : str.split(x, ',')[0])
 df['IsAlone'] = 0
df.loc[df.FamilySize == 1, 'IsAlone'] = 1
le = LabelEncoder()
df['NameLength'] = df.Name.apply(lambda x : len(x))
df['NameLengthBin'] = pd.qcut(df.NameLength, 5)
 df['NameLengthBinCode'] = le.fit_transform(df.NameLengthBin)
 df['Embarked'] = pd.Categorical(df.Embarked ).codes
df = pd.concat([df, pd.get_dummies(df.Sex)], axis = 1)
 table_ticket = pd.DataFrame(df.Ticket.value_counts())
 table_ticket.rename(columns = {'Ticket' : 'TicketNum'}, inplace = True)
 table_ticket['TicketId'] = pd.Categorical(table_ticket.index ).codes
table_ticket.loc[table_ticket.TicketNum < 3, 'TicketId'] = -1
df = pd.merge(left = df, right = table_ticket, left_on = 'Ticket', right_index = True, how = 'left', sort = False)
 df['TicketCode'] = list(pd.cut(df.TicketId, bins = [-2, 0, 500, 1000], labels = [0, 1, 2]))
regex = re.compile('\s*(\w+)\s*')
 df['CabinNum'] = df.Cabin.apply(lambda x : len(regex.findall(x)) )
lg = LogisticRegression(solver='lbfgs', random_state=18)
 lg.fit(X_train, y_train)
 logistic_prediction = lg.predict(X_test)
 score = metrics.accuracy_score(y_test, logistic_prediction)
 print(score)
 confusion_matrix(y_test,logistic_prediction )<train_model>
from sklearn.ensemble import ExtraTreesRegressor
data_dmatrix = xgb.DMatrix(data=Xn,label=y)
 xgc = xgb.XGBClassifier(objective ='reg:logistic', colsample_bytree = 0.15,
learning_rate = 0.1,
max_depth = 20, alpha = 12, n_estimators = 700)
 xgc.fit(X,y )<predict_on_test>
classers = ['Fare','Parch','Pclass','SibSp','TitleCat', 'CabinCat','female','male', 'Embarked', 'FamilySize', 'IsAlone', 'NameLengthBinCode','TicketNum','TicketCode']
etr = ExtraTreesRegressor(n_estimators = 200, random_state = 0)
 age_X_train = df[classers][df.Age.notnull() ]
age_y_train = df.Age[df.Age.notnull() ]
age_X_test = df[classers][df.Age.isnull() ]
etr.fit(age_X_train, np.ravel(age_y_train))
age_pred = etr.predict(age_X_test)
 df.loc[df.Age.isnull() , 'Age'] = age_pred
pred_train=xgc.predict(X)
 pred_train.sum()<predict_on_test>
childAge = 18
def getIdentity(passenger):
age, sex = passenger
if age < childAge:
return 'child'
elif sex == 'male':
return 'male_adult'
else:
return 'female_adult'
df = pd.concat([df, pd.DataFrame(df[['Age', 'Sex']].apply(getIdentity, axis = 1), columns = ['Identity'])], axis = 1)
 df = pd.concat([df, pd.get_dummies(df.Identity)], axis = 1 )
pred_test=xgc.predict(Xt)
 pred_test.sum()<create_dataframe>
DEFAULT_SURVIVAL_VALUE = 0.5
df['FamilySurvival'] = DEFAULT_SURVIVAL_VALUE
for _, grp_df in df.groupby(['FamilyName', 'Fare']):
if len(grp_df)!= 1 :
for index, row in grp_df.iterrows() :
smax = grp_df.drop(index ).Survived.max()
smin = grp_df.drop(index ).Survived.min()
pid = row.PassengerId
if smax == 1:
df.loc[df.PassengerId == pid, 'FamilySurvival'] = 1.0
elif smin == 0:
df.loc[df.PassengerId == pid, 'FamilySurvival'] = 0.0
for _, grp_df in df.groupby(['Ticket']):
if len(grp_df != 1):
for index, row in grp_df.iterrows() :
if(row.FamilySurvival == 0.0 or row.FamilySurvival == 0.5):
smax = grp_df.drop(index ).Survived.max()
smin = grp_df.drop(index ).Survived.min()
pid = row.PassengerId
if smax == 1:
df.loc[df.PassengerId == pid, 'FamilySurvival'] = 1.0
elif smin == 0:
df.loc[df.PassengerId == pid, 'FamilySurvival'] = 0.0
df.FamilySurvival.value_counts()
yt=pd.DataFrame(pred_test)
 yt.index=yt.index+1
yt<save_to_csv>
df['FareBin'] = pd.qcut(df.Fare, 5)
 le = LabelEncoder()
df['FareBinCode'] = le.fit_transform(df.FareBin )
test=pd.read_csv('.. /input/equipfails/equip_failures_test_set.csv',na_values='na')
 df= pd.DataFrame()
df['id'] = test['id']
df['target'] = pred_test
df.to_csv('submission2.csv', index=False )<save_to_csv>
df['AgeBin'] = pd.qcut(df.Age, 4)
 le = LabelEncoder()
df['AgeBinCode'] = le.fit_transform(df.AgeBin )
file_name='submision.csv'
yt.to_csv(file_name,index=True)
<load_pretrained>
from sklearn.preprocessing import MinMaxScaler, StandardScaler
filename = 'Final_Model.mod'
pickle.dump(xgc, open(filename, 'wb'))<set_options>
target = data_train['Survived'].values
select_features = ['AgeBinCode', 'Embarked', 'FareBinCode', 'Parch', 'Pclass', 'SibSp', 'CabinCat', 'TitleCat', 'FamilySize', 'IsAlone', 'FamilySurvival', 'NameLengthBinCode', 'female', 'male', 'TicketNum', 'TicketCode', 'CabinNum', 'child', 'female_adult', 'male_adult']
sns.set()
%matplotlib inline
warnings.filterwarnings('ignore')
 %load_ext tensorboard.notebook
pd.set_option('display.max_columns', None)
<load_from_csv>
scaler = StandardScaler()
df_scaled = scaler.fit_transform(df[select_features])
 train = df_scaled[0:891].copy()
test = df_scaled[891:].copy()
odf=pd.read_csv('/kaggle/input/equipfails/equip_failures_training_set.csv',index_col=0)
 odft=pd.read_csv('/kaggle/input/equipfails/equip_failures_test_set.csv',index_col=0 )<data_type_conversions>
from sklearn.feature_selection import SelectKBest, f_classif
df=odf.replace({'na':-999999})
 Xt=odft.replace({'na':-999999})
 Xt=Xt.astype(float)
 df=df.astype(float)
 df['target']=df['target'].astype(int )<prepare_x_and_y>
selector = SelectKBest(f_classif, len(select_features))
selector.fit(train, target)
 scores = -np.log10(selector.pvalues_)
 indices = np.argsort(scores)[::-1]
print('Features importance:')
 for i in range(len(scores)) :
print('%.2f %s' %(scores[indices[i]], select_features[indices[i]]))
X=df.iloc[:,1:]
y=df.iloc[:,0]
Xn=normalize(X)
 Xtn=normalize(Xt )<split>
kf = KFold(n_splits = 5, random_state = 1 )
X_train, X_test, y_train, y_test = train_test_split(Xn, y, test_size=0.2, random_state=8 )<compute_train_metric>
rfc_parameters = {'max_depth' : [5], 'n_estimators' : [500], 'min_samples_split' : [9], 'random_state' : [1], 'n_jobs' : [-1]}
rfc = RandomForestClassifier()
clf_rfc = GridSearchCV(rfc, rfc_parameters, n_jobs = -1, cv = kf, scoring = 'roc_auc')
 clf_rfc.fit(train, target )
my_class = ExtraTreesClassifier(random_state=0)
 my_class.fit(X_train, y_train)
 y_pred= clf.predict(X_test)
 print('accuracy: {}'.format(accuracy_score(y_test, y_pred)))
 print(f'F1: {f1_score(y_test,y_pred)}')
 confusion_matrix(y_test,y_pred )<compute_test_metric>
rfc2_parameters = {'max_depth' : [2, 5, 8, 10, 20, 50], 'n_estimators' : [10, 50, 100, 200, 500, 1000, 2000], 'min_samples_split' : [2, 3, 5, 9, 20]}
rfc2 = RandomForestClassifier(random_state = 1, n_jobs = -1)
 clf_rfc2 = RandomizedSearchCV(rfc2, rfc2_parameters, n_jobs = -1, cv = kf, scoring = 'roc_auc')
 clf_rfc2.fit(train, target )
my_class = AdaBoostClassifier(random_state=0)
 my_class.fit(X_train, y_train)
 y_pred= my_class.predict(X_test)
 print('accuracy: {}'.format(accuracy_score(y_test, y_pred)))
 print(f'F1: {f1_score(y_test,y_pred)}')
 confusion_matrix(y_test,y_pred )<compute_train_metric>
importance = clf_rfc.best_estimator_.feature_importances_
indices = np.argsort(importance)[::-1]
print(clf_rfc.best_score_)
 print(clf_rfc.score(train, target))
print(clf_rfc.best_params_)
 print('
Feature importance:')
 for i in range(len(select_features)) :
print('%.2f %s' %(importance[indices[i]], select_features[indices[i]]))
lg = LogisticRegression(solver='lbfgs', random_state=18)
 lg.fit(X_train, y_train)
 logistic_prediction = lg.predict(X_test)
 score = metrics.accuracy_score(y_test, logistic_prediction)
 print(score)
 confusion_matrix(y_test,logistic_prediction )<train_model>
lr_paramaters = {'C' : [0.05, 0.1, 0.2], 'random_state' : [1]}
lr = LogisticRegression()
clf_lr = GridSearchCV(lr, lr_paramaters, n_jobs = -1, cv = kf, scoring = 'roc_auc')
 clf_lr.fit(train, target )
data_dmatrix = xgb.DMatrix(data=Xn,label=y)
 xgc = xgb.XGBClassifier(objective ='reg:logistic', colsample_bytree = 0.2,
learning_rate = 0.1,
max_depth = 20, alpha = 10, n_estimators = 700)
 xgc.fit(X,y )<predict_on_test>
print(clf_lr.best_score_)
 print(clf_lr.score(train, target))
print(clf_lr.best_params_ )
pred_train=xgc.predict(X)
 pred_train.sum()<predict_on_test>
svc_paramaters = {'C' : [5.5, 6, 6.5], 'kernel' : ['linear', 'rbf'], 'gamma' : ['auto', 'scale'], 'random_state' : [1]}
svc = SVC()
clf_svc = GridSearchCV(svc, svc_paramaters, n_jobs = -1, cv = kf, scoring = 'roc_auc')
 clf_svc.fit(train, target )
pred_test=xgc.predict(Xt)
 pred_test.sum()<create_dataframe>
print(clf_svc.best_score_)
 print(clf_svc.score(train, target))
print(clf_svc.best_params_ )
yt=pd.DataFrame(pred_test)
 yt.index=yt.index+1
yt<save_to_csv>
gbdt_parameters = {'subsample' : [1], 'min_samples_leaf' : [3], 'learning_rate' : [0.1], 'n_estimators' : [50], 'min_samples_split' : [2], 'max_depth' : [3], 'random_state' : [1]}
gbdt = GradientBoostingClassifier()
clf_gbdt = GridSearchCV(gbdt, gbdt_parameters, n_jobs = -1, cv = kf, scoring = 'roc_auc')
 clf_gbdt.fit(train, target )
test=pd.read_csv('.. /input/equipfails/equip_failures_test_set.csv',na_values='na')
 df= pd.DataFrame()
df['id'] = test['id']
df['target'] = pred_test
df.to_csv('submission2.csv', index=False )<save_to_csv>
print(clf_gbdt.best_score_)
 print(clf_gbdt.score(train, target))
print(clf_gbdt.best_params_ )
file_name='submision.csv'
yt.to_csv(file_name,index=True)
<load_pretrained>
xgb_paramaters = {'subsample' : [0.7], 'min_child_weight' : [1], 'max_depth' : [3], 'learning_rate' : [0.1], 'n_estimators' : [100], 'n_jobs' : [-1], 'random_state' : [1]}
xgb = XGBClassifier()
clf_xgb = GridSearchCV(xgb, xgb_paramaters, n_jobs = -1, cv = kf, scoring = 'roc_auc')
 clf_xgb.fit(train, target )
filename = 'Final_Model.mod'
pickle.dump(xgc, open(filename, 'wb'))<load_from_csv>
print(clf_xgb.best_score_)
 print(clf_xgb.score(train, target))
print(clf_xgb.best_params_ )
df2 = pd.read_csv(".. /input/train.csv", parse_dates=[0])
 test = pd.read_csv(".. /input/test.csv", parse_dates=[0] )<create_dataframe>
prediction = clf_rfc2.predict(test )
df3 = df2.copy()
test3 = test.copy()<data_type_conversions>
submission = pd.DataFrame({'Survived' : prediction}, index = data_test.PassengerId)
 submission.to_csv('submission.csv', index_label = ['PassengerId'] )