Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<set_options>
warnings.filterwarnings('ignore')
gc.collect()<load_pretrained>
train = pd.read_csv(".. /input/train.csv")
 taille_train = len(train)
 labels = train['Survived']
test = pd.read_csv(".. /input/test.csv")
 all_data = pd.concat([train, test],ignore_index=True, sort='True' )
tfms = tfms_from_model(resnet34, sz, aug_tfms=transforms_side_on, max_zoom=1.1 )<choose_model_class>
all_data.isnull().sum()
arch=resnet34<load_pretrained>
train[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
data = ImageClassifierData.from_paths(PATH, tfms=tfms, test_name='test')
 learn = ConvLearner.pretrained(arch, data, precompute=True )<train_model>
train[["Sex", "Survived"]].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
learn.fit(1e-2, 1 )<set_options>
def get_name(name):
lastName = name.split(',')
 title = lastName[1].split('.')
 if(lastName and title):
return lastName[0], title[0], title[1]
return ""
r = all_data['Name'].apply(get_name)
 LastNames, titles, Firstames = [], [], []
for i in range(len(r)) :
LastNames.append(r[i][0])
 titles.append(r[i][1])
 Firstames.append(r[i][2])
 all_data['LastName'] = LastNames
all_data['Title'] = titles
all_data['FirstName'] = Firstames
learn.precompute=False<train_model>
def combine_title(title):
title = title.replace(" ","")
 title = title.replace('Mlle', 'Miss')
 title = title.replace('Ms', 'Miss')
 title = title.replace('Mme', 'Mrs')
 if title in ['Lady', 'theCountess', 'Don', 'Sir', 'Jonkheer', 'Dona']:
return "Royal"
elif title in ['Capt', 'Col', 'Dr', 'Major', 'Rev'] :
return 'Other'
else :
return title
all_data['Title'] = all_data['Title'].apply(combine_title )
learn.fit(1e-2, 3, cycle_len=1 )<set_options>
all_data[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
gc.collect()<define_search_space>
all_data['FamilySize'] = all_data['SibSp'] + all_data['Parch'] + 1
lr=np.array([1e-4,1e-3,1e-2] )<train_model>
all_data['IsAlone'] = all_data[['FamilySize']].applymap(lambda x: 0 if x>1 else 1 )
learn.fit(lr, 3, cycle_len=1, cycle_mult=2 )<feature_engineering>
all_data['Age'].fillna(all_data['Age'].median() , inplace = True )
log_preds,y = learn.TTA()
probs = np.mean(np.exp(log_preds),0 )<compute_test_metric>
fare_moyen = all_data[['Pclass', 'Fare']].groupby(['Pclass'], as_index=False ).mean()
fare_moyen
accuracy_np(probs, y )<prepare_output>
all_data.loc[(all_data['Fare'].isnull())|(all_data['Fare']==0)].Fare
preds = np.argmax(probs, axis=1)
 probs = probs[:,1]<predict_on_test>
for i in range(3):
all_data.loc[(all_data['Fare']==0)&(all_data['Pclass']==(i+1)) ,'Fare'] = fare_moyen.Fare[i]
all_data['Fare'] = all_data['Fare'].fillna(fare_moyen.Fare[2] )
temp = learn.predict(is_test=True )<prepare_output>
all_data.loc[(all_data['Embarked'].isnull())]
pred_test = np.argmax(temp, axis=1 )<prepare_output>
all_data.loc[(all_data['Cabin']=='B28')]
probs = np.exp(temp[:,1] )<create_dataframe>
all_data['Embarked'] = all_data['Embarked'].fillna('S' )
submission = pd.DataFrame({'id':os.listdir(f'{PATH}test'), 'label':probs} )<feature_engineering>
all_data['HasCabin'] = all_data[['Cabin']].applymap(lambda x: 0 if pd.isnull(x)else 1 )
submission['id'] = submission['id'].map(lambda x: x.split('.')[0] )<data_type_conversions>
all_data['Title'] = all_data['Title'].map({'Master': 1, 'Miss': 2, 'Mr': 3, 'Mrs':4, 'Other': 5, 'Royal':6} ).astype(int)
 all_data['Embarked'] = all_data['Embarked'].map({'C': 0, 'Q': 1, 'S': 2} ).astype(int)
 all_data['Sex'] = all_data['Sex'].map({'female': 1, 'male': 0} ).astype(int )
submission['id'] = submission['id'].astype(int )<sort_values>
all_data = all_data.drop(columns=['Name','LastName', 'FirstName','Ticket','Cabin', 'Parch', 'SibSp', 'PassengerId'])
 all_data.head()
submission = submission.sort_values('id' )<save_to_csv>
train = all_data.iloc[0:taille_train]
test = all_data.iloc[taille_train:]
submission.to_csv('.. /working/output.csv', index=False )<categorify>
train[["FamilySize", "Survived"]].groupby(["FamilySize"], as_index=False ).mean().sort_values(by='Survived', ascending=False )
def OHEncode(df, cols):
for x in cols:
x_ohe = pd.get_dummies(df[x], prefix=x)
 df = pd.concat([df, x_ohe], axis=1)
 df = df.drop([x], axis=1)
 return df<define_variables>
train = train.drop(columns=['Survived'])
 test = test.drop(columns=['Survived'])
cols = ['platform']<choose_model_class>
X_train, X_test, y_train, y_test = train_test_split(train, labels, test_size=0.3 )
testing=False
chunksize=50000
clf = LogisticRegression()<load_from_csv>
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, y_train)
 Y_pred = random_forest.predict(X_test)
 score_rf_train = random_forest.score(X_train, y_train)
 score_rf_test = random_forest.score(X_test, y_test)
 print("score de training : "+ str(score_rf_train)+ " score de testing : " + str(score_rf_test))
train = pd.read_csv(".. /input/train-featured/clicks_events_full.csv", iterator=True,chunksize=chunksize )<categorify>
def logloss(true_label, predicted, eps=1e-15):
p = np.clip(predicted, eps, 1 - eps)
 if true_label == 1:
return -log(p)
 else:
return -log(1 - p )
print('Chunks training')
 for chunk in train:
chunk = chunk.drop(['geo_location','dayofweek','hour'], axis=1)
 chunk = OHEncode(chunk,cols)
 predictors=[x for x in chunk.columns if x not in ['display_id','clicked']]
chunk=chunk.fillna(0.0)
 clf.fit(chunk[predictors], chunk["clicked"])
 if testing:
break
train=''<load_from_csv>
rf = RandomForestClassifier(n_estimators = 200)
 rf.fit(X_train, y_train)
 y_proba_test = rf.predict_proba(X_test)
 y_proba_train = rf.predict_proba(X_train)
 test_loss = log_loss(y_test, y_proba_test, eps = 1e-15, normalize = True, sample_weight = None, labels = None)
 print("test cross entropy = " + str(test_loss))
train_loss = log_loss(y_train, y_proba_train, eps = 1e-15, normalize = True, sample_weight = None, labels = None)
 print("train cross entropy = " + str(train_loss))
test = pd.read_csv(".. /input/test-featured/test_events_full.csv",iterator=True,chunksize=chunksize )<categorify>
dtrain = xgb.DMatrix(X_train, label = y_train)
 dtest = xgb.DMatrix(X_test, label = y_test)
 watchlist = [(dtrain, 'train'),(dtest, 'valid')]
print('Testing')
 predY=[]
for chunk in test:
init_chunk_size=len(chunk)
 chunk = chunk.drop(['geo_location','dayofweek','hour'], axis=1)
 chunk = OHEncode(chunk,cols)
 chunk=chunk.fillna(0.0)
 chunk_pred=list(clf.predict_proba(chunk[predictors] ).astype(float)[:,1])
 predY += chunk_pred
if testing:
break
print('Done Testing' )<load_from_csv>
param = {'max_depth': 2, 'eta':0.02, 'subsample' : 0.6,'base_score' : 0.2, "objective" : "binary:logistic", "eval_metric": "logloss"}
bst = xgb.train(param, dtrain, 10000, watchlist, early_stopping_rounds=150, verbose_eval=50)
 y_pred = bst.predict(dtest)
 loss = log_loss(y_test, y_pred, eps = 1e-15, normalize = True, sample_weight = None, labels = None)
 print("cross entropy = " + str(loss))
print('Preparing for Submission')
 test=''
test= pd.read_csv('.. /input/outbrain-click-prediction/clicks_test.csv' )<concatenate>
X_train, X_test, y_train, y_test = train_test_split(train, labels, test_size=0.3)
 X_train['Survived'] = y_train
X_test['Survived'] = y_test
def modelfit(alg, dtrain, dtest, predictors, useTrainCV=True, cv_folds=5, early_stopping_rounds=50):
if useTrainCV:
xgb_param = alg.get_xgb_params()
xgtrain = xgb.DMatrix(dtrain[predictors].values, label=dtrain['Survived'].values)
 cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params() ['n_estimators'], nfold=cv_folds,
metrics='logloss', early_stopping_rounds=early_stopping_rounds)
 alg.set_params(n_estimators=cvresult.shape[0])
 alg.fit(dtrain[predictors], dtrain['Survived'],eval_metric='logloss')
 dtrain_predictions = alg.predict(dtrain[predictors])
 dtrain_predprob = alg.predict_proba(dtrain[predictors])[:,1]
dtest_predictions = alg.predict(dtest[predictors])
 dtest_predprob = alg.predict_proba(dtest[predictors])[:,1]
print("
Rapport d'informations")
 print("Accuracy sur le set d'entrainement: %.4g" % metrics.accuracy_score(dtrain['Survived'].values, dtrain_predictions))
print("Accuracy sur le set de test: %.4g" % metrics.accuracy_score(dtest['Survived'].values, dtest_predictions))
loss = log_loss(dtrain['Survived'], dtrain_predprob, eps = 1e-15, normalize = True, sample_weight = None, labels = None)
 print("cross entropy(train)= " + str(loss))
loss = log_loss(dtest['Survived'], dtest_predprob, eps = 1e-15, normalize = True, sample_weight = None, labels = None)
 print("cross entropy(test)= " + str(loss))
plot_importance(alg)
 pyplot.show()
return alg
results=pd.concat(( test,pd.DataFrame(predY)) ,axis=1,ignore_index=True)
 print(results.head(10))<rename_columns>
predictors = [x for x in X_train.columns if x not in ['Survived']]
xgb1 = XGBClassifier(
 learning_rate =0.1,
n_estimators=1000,
max_depth=2,
min_child_weight=1,
gamma=0,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27 )
results.columns = ['display_id','ad_id','clicked']
print(results.head(10))<sort_values>
m1 = modelfit(xgb1, X_train, X_test, predictors )
results = results.sort_values(by=['display_id','clicked'], ascending=[True, False])
 results = results.reset_index(drop=True )<create_dataframe>
param_test1 = {
'n_estimators':[300,500],
'max_depth':[4,5,6],
'min_child_weight':[4,5,6],
}
results2 = results.copy()<groupby>
gsearch1 = GridSearchCV(estimator = XGBClassifier(learning_rate = 0.01, n_estimators=500, max_depth=5,
min_child_weight=4, gamma=0, subsample=0.8, colsample_bytree=0.8,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27),
param_grid = param_test1, scoring='neg_log_loss',n_jobs=4,iid=False, cv=None)
 gsearch1.fit(X_train[predictors], X_train['Survived'])
 gsearch1.best_params_, gsearch1.best_score_
submission_data = results2.groupby('display_id' ).ad_id.apply(lambda x: " ".join(map(str,x)) ).reset_index()<save_to_csv>
param_test1 = {
'learning_rate':[0.01,0.05,0.025],
'n_estimators':[500,520,540],
'max_depth':[5],
'min_child_weight': [4]
}
submission_data.to_csv('submission_feature_removed.csv', index=False )<set_options>
gsearch1 = GridSearchCV(estimator = XGBClassifier(learning_rate = 0.01, n_estimators=500, max_depth=5,
min_child_weight=4, gamma=0, subsample=0.8, colsample_bytree=0.8,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27),
param_grid = param_test1, scoring='neg_log_loss',n_jobs=4,iid=False, cv=None)
 gsearch1.fit(X_train[predictors], X_train['Survived'])
 gsearch1.best_params_, gsearch1.best_score_
K.tensorflow_backend._get_available_gpus()<set_options>
param_test1 = {
'learning_rate':[0.01],
'n_estimators':[520],
'max_depth':[5],
'min_child_weight': [4],
'gamma':[0.0, 0.05, 0.1],
'reg_alpha':[1e-5, 1e-2, 0.1, 1]
}
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'
pd.set_option('display.max_columns', 1000)
<set_options>
gsearch1 = GridSearchCV(estimator = XGBClassifier(learning_rate = 0.01, n_estimators=520, max_depth=5,
min_child_weight=4, gamma=0, subsample=0.8, colsample_bytree=0.8,
objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27),
param_grid = param_test1, scoring='neg_log_loss',n_jobs=4,iid=False, cv=None)
 gsearch1.fit(X_train[predictors], X_train['Survived'])
 gsearch1.best_params_, gsearch1.best_score_
pd.set_option('display.max_columns', 50 )<load_from_csv>
xgbfinal = XGBClassifier(
 learning_rate =0.01,
n_estimators=520,
max_depth=5,
min_child_weight=4,
gamma=0,
subsample=0.8,
colsample_bytree=0.8,
reg_alpha=0.01,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27 )
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )<load_from_csv>
mfinal = modelfit(xgbfinal, X_train, X_test, predictors )
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )<concatenate>
y_pred = mfinal.predict(test )
df_all = pd.concat([train, test], ignore_index=True, sort=False)
 df_all.loc[np.isfinite(df_all.AdoptionSpeed), 'is_train'] = 1
df_all.loc[~np.isfinite(df_all.AdoptionSpeed), 'is_train'] = 0<define_variables>
testid = pd.read_csv(".. /input/test.csv" )
all_columns = []<define_variables>
submission = pd.DataFrame({
"PassengerId": testid["PassengerId"],
"Survived": y_pred
} )
<prepare_output><EOS> 
submission.to_csv("Titanic.csv", index = False )