Kaggle competition: Titanic - Machine Learning from Disaster
if TRAINING:
for i in range(3, -1, -1):
y_train[:, i] = np.logical_or(y_train[:, i], y_train[:, i + 1])
 y_val[:, i] = np.logical_or(y_val[:, i], y_val[:, i + 1])
 print("Multilabel version:", y_train.sum(axis=0))<normalization>
train_data = '.. /input/titanic/train.csv'
test_data = '.. /input/titanic/test.csv'
if TRAINING:
for i in tqdm(range(len(x_val))):
x_val[i] = cv2.resize(x_val[i, 20: -20, 20: -20, :],(IMG_SIZE, IMG_SIZE))
for i in tqdm(range(len(x_train))):
x_train[i] = cv2.resize(x_train[i, 20: -20, 20: -20, :],(IMG_SIZE, IMG_SIZE))<define_variables>
train_set = pd.read_csv(train_data)
 test_set = pd.read_csv(test_data )
class MixupGenerator() :
def __init__(self, X_train, y_train, seq=None, batch_size=32, alpha=0.2, shuffle=True, datagen=None):
self.X_train = X_train
self.y_train = y_train
self.batch_size = batch_size
self.alpha = alpha
self.shuffle = shuffle
self.sample_num = len(X_train)
 self.datagen = datagen
self.seq = seq
def __call__(self):
while True:
indexes = self.__get_exploration_order()
itr_num = int(len(indexes)//(self.batch_size * 2))
for i in range(itr_num):
batch_ids = indexes[i * self.batch_size * 2:(i + 1)* self.batch_size * 2]
X, y = self.__data_generation(batch_ids)
 yield X, y
def __get_exploration_order(self):
indexes = np.arange(self.sample_num)
 if self.shuffle:
np.random.shuffle(indexes)
 return indexes
def __data_generation(self, batch_ids):
_, h, w, c = self.X_train.shape
l = np.random.beta(self.alpha, self.alpha, self.batch_size)
 X_l = l.reshape(self.batch_size, 1, 1, 1)
 y_l = l.reshape(self.batch_size, 1)
 X1 = self.X_train[batch_ids[:self.batch_size]]
X2 = self.X_train[batch_ids[self.batch_size:]]
X = X1 * X_l + X2 *(1 - X_l)
 if self.datagen:
for i in range(self.batch_size):
X[i] = self.datagen.random_transform(X[i])
 X[i] = self.datagen.standardize(X[i])
 if self.seq:
X = X.astype(np.float32)
 X /= np.max(X)
 X *= 255
X = X.astype(np.uint8)
 X = self.seq.augment_images(X)
 if isinstance(self.y_train, list):
y = []
for y_train_ in self.y_train:
y1 = y_train_[batch_ids[:self.batch_size]]
y2 = y_train_[batch_ids[self.batch_size:]]
y.append(y1 * y_l + y2 *(1 - y_l))
else:
y1 = self.y_train[batch_ids[:self.batch_size]]
y2 = self.y_train[batch_ids[self.batch_size:]]
y = y1 * y_l + y2 *(1 - y_l)
 return X, y
def get_sample(self):
return self.__data_generation(self.__get_exploration_order() [i * self.batch_size * 2:(i + 1)* self.batch_size * 2] )<train_model>
train_set.isnull().sum(axis=0 )
BATCH_SIZE = 32
def create_datagen() :
return ImageDataGenerator(
 preprocessing_function=seq.augment_image 
)
 if TRAINING:
data_generator = create_datagen().flow(x_train, y_train, batch_size=BATCH_SIZE)
 mixup_generator = MixupGenerator(x_train, y_train, seq=seq, batch_size=BATCH_SIZE, alpha=0.2 )()<define_variables>
%matplotlib inline
sns.set_style('whitegrid')
 warnings.filterwarnings('ignore' )
true_labels = np.array([1, 0, 1, 1, 0, 1])
 pred_labels = np.array([1, 0, 0, 0, 0, 1] )<compute_test_metric>
train_set.isnull().sum()
accuracy_score(true_labels, pred_labels )<compute_test_metric>
train_set['Cabin'].isnull().sum()
cohen_kappa_score(true_labels, pred_labels )<predict_on_test>
train_set['Cabin'].value_counts().head()
class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):
X_val, y_val = self.validation_data[:2]
y_val =(y_val.sum(axis=1)- 1 ).clip(0, 4)
 y_pred = self.model.predict(X_val)> 0.5
y_pred =(y_pred.astype(int ).sum(axis=1)- 1 ).clip(0, 4)
 _val_kappa = cohen_kappa_score(
 y_val,
y_pred,
weights='quadratic' 
)
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {_val_kappa:.4f}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model.save('model.h5')
 return<load_pretrained>
merged = pd.concat([train_set,test_set], sort = False)
 merged.head(3 )
base_model = Xception(
 weights=None,
include_top=False,
input_shape=(224,224,3)
)
 base_model.load_weights(".. /input/keras-pretrained-models/xception_weights_tf_dim_ordering_tf_kernels_notop.h5" )<choose_model_class>
merged['Cabin'].value_counts().head(3 )
def build_model() :
model = Sequential()
model.add(base_model)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.5))
model.add(layers.Dense(5, activation='sigmoid'))
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(lr=0.00005),
metrics=['accuracy'] 
)
 return model<train_on_grid>
merged['Cabin'].fillna('X', inplace=True )
if TRAINING:
kappa_metrics = Metrics()
rlr = callbacks.ReduceLROnPlateau(factor=0.5, patience=4, verbose=1)
 es = callbacks.EarlyStopping(patience=10, verbose=1, mode="min")
 history = model.fit_generator(
 data_generator,
steps_per_epoch=x_train.shape[0] / BATCH_SIZE,
epochs=200,
validation_data=(x_val, y_val),
callbacks=[kappa_metrics, rlr, es]
)<save_to_csv>
merged['Title'] = merged['Name'].str.extract('([A-Za-z]+)\.')
 merged['Title'].head()
if TRAINING:
with open('history.json', 'w')as f:
json.dump(str(history.history), f)
 history_df = pd.DataFrame(history.history)
 history_df[['loss', 'val_loss']].plot()
history_df[['acc', 'val_acc']].plot()<data_type_conversions>
merged['Title'].value_counts()
<save_to_csv>
merged['Title'].replace(to_replace = ['Dr', 'Rev', 'Col', 'Major', 'Capt'], value = 'Officer', inplace=True)
 merged['Title'].replace(to_replace = ['Dona', 'Jonkheer', 'Countess', 'Sir', 'Lady', 'Don'], value = 'Aristocrat', inplace = True)
 merged['Title'].replace({'Mlle':'Miss', 'Ms':'Miss', 'Mme':'Mrs'}, inplace = True )
if TRAINING:
weights_path = 'model.h5'
else:
weights_path = '.. /input/weights/model.h5'
model.load_weights(weights_path)
 y_test = model.predict(x_test)> 0.5
y_test =(y_test.astype(int ).sum(axis=1)- 1 ).clip(0, 4)
 test_df['diagnosis'] = y_test
test_df.to_csv('submission.csv',index=False )<set_options>
merged['SibSp'].value_counts()
%reload_ext autoreload
%autoreload 2
%matplotlib inline
%matplotlib inline 
<define_variables>
merged['Parch'].value_counts()
package_dir = '.. /input/efficientnet/efficientnet_pytorch'
sys.path.insert(0, package_dir)
<load_pretrained>
merged['Family_size'] = merged.SibSp + merged.Parch + 1
merged['Family_size'].value_counts()
md_ef = EfficientNet.from_pretrained('efficientnet-b5', num_classes=1 )<load_from_csv>
merged['Family_size'].replace(to_replace = [1], value = 'single', inplace = True)
 merged['Family_size'].replace(to_replace = [2,3], value = 'small', inplace = True)
 merged['Family_size'].replace(to_replace = [4,5], value = 'medium', inplace = True)
 merged['Family_size'].replace(to_replace = [6, 7, 8, 11], value = 'large', inplace = True )
def get_df() :
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 return df, test_df
df, test_df = get_df()<feature_engineering>
merged['Family_size'].value_counts()
bs = 64
sz = 224
tfms = get_transforms(do_flip=True,flip_vert=True )<compute_test_metric>
ticket = []
for x in list(merged['Ticket']):
if x.isdigit() :
ticket.append('N')
 else:
ticket.append(x.replace('.','' ).replace('/','' ).strip().split(' ')[0])
 merged['Ticket'] = ticket
def qk(y_pred, y):
return torch.tensor(cohen_kappa_score(torch.round(y_pred), y, weights='quadratic'), device='cuda:0' )<load_pretrained>
merged['Ticket'].value_counts()
learn = Learner(data,
md_ef,
metrics = [qk],
model_dir="models" ).to_fp16()
learn.data.add_test(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png'))<train_model>
merged['Ticket'] = merged['Ticket'].apply(lambda x: x[0])
 merged['Ticket'].value_counts()
learn.fit_one_cycle(10,1e-3 )<compute_train_metric>
outliers(merged['Fare'] )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<save_to_csv>
merged.isnull().sum()
def run_subm(learn=learn, coefficients=[0.5, 1.5, 2.5, 3.5]):
opt = OptimizedRounder()
preds,y = learn.get_preds(DatasetType.Test)
 tst_pred = opt.predict(preds, coefficients)
 test_df.diagnosis = tst_pred.astype(int)
 test_df.to_csv('submission.csv',index=False)
 print('done' )<set_options>
merged['Embarked'].value_counts()
%reload_ext autoreload
%autoreload 2
%matplotlib inline 
<set_options>
merged['Embarked'].fillna(value = 'S', inplace=True )
warnings.filterwarnings('always')
 warnings.filterwarnings('ignore')
 %matplotlib inline
style.use('fivethirtyeight')
 sns.set(style='whitegrid', color_codes=True)
<set_options>
merged['Fare'].fillna(value= merged['Fare'].median() , inplace=True )
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(42 )<feature_engineering>
df = merged.loc[:, ['Sex', 'Pclass', 'Embarked', 'Title', 'Family_size', 'Parch', 'SibSp', 'Cabin', 'Ticket']]
LE = LabelEncoder()
df = df.apply(LE.fit_transform)
 df.head(5)
temp = vision.data.open_image 
<load_from_csv>
df['Age'] = merged['Age']
df.head(2 )
def get_df() :
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 return df, test_df
df, test_df = get_df()
bs = 16
sz = 380 
<import_modules>
df = df.set_index('Age' ).reset_index()
df.head(2 )
from fastai import *
from fastai.vision import *
from torchvision.models import *
import torch
import torch.optim as optim
from fastai.vision.models import *
from fastai.vision.learner import model_meta
from fastai.callbacks import *<init_hyperparams>
merged['Age'] = merged.groupby(['Title', 'Pclass'])['Age'].transform(lambda x: x.fillna(x.median()))
aptos19_stats =([0.42, 0.22, 0.075], [0.27, 0.15, 0.081])
 tsfm1 = get_transforms(do_flip=False, flip_vert=False, max_rotate=0., max_zoom=0., max_warp=0.0, max_lighting=0.,p_affine=0.)
data =(ImageList.from_df(df=df,path='./',cols='path')
.split_by_rand_pct(0.1)
.label_from_df(cols='diagnosis',label_cls=FloatList)
.transform(tsfm1,size=380,)
.databunch(bs=bs,num_workers=4)
.normalize(imagenet_stats)
 )<define_variables>
merged.loc[:, ['Pclass', 'Sex', 'Embarked', 'Cabin', 'Title', 'Family_size', 'Ticket']] = merged.loc[:, ['Pclass', 'Sex', 'Embarked', 'Cabin', 'Title', 'Family_size', 'Ticket']].astype('category')
 merged['Survived'] = merged['Survived'].dropna().astype('int' )
data.show_batch(3,(5,5))<load_pretrained>
merged.drop(columns = ['Name', 'Age','SibSp', 'Parch','Fare'], inplace = True, axis = 1)
 merged.columns
md_ef = EfficientNet.from_pretrained('efficientnet-b5', num_classes=1 )<compute_test_metric>
merged = pd.get_dummies(merged, drop_first=True)
 merged.head(2 )
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0' )<load_pretrained>
train = merged.iloc[:891, :]
test = merged.iloc[891:, :]
learn = Learner(data,
md_ef,
metrics = [quadratic_kappa]
,callback_fns=[BnFreeze,partial(SaveModelCallback, monitor='quadratic_kappa', name='best_accuracy')] ).to_fp16()
learn.data.add_test(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png')) 
<define_variables>
train = train.drop(columns = ['PassengerId'], axis = 1)
 test = test.drop(columns = ['Survived'], axis = 1 )
learn.model_dir="/kaggle/working/models"<load_pretrained>
X_train = train.drop(columns = ['Survived'], axis = 1)
 y_train = train['Survived']
X_test = test.drop("PassengerId", axis = 1 ).copy()
learn.load('modelk');<find_best_params>
LR = LogisticRegression()
KNN = KNeighborsClassifier()
DT = DecisionTreeClassifier()
RF = RandomForestClassifier()
SVM = SVC(gamma='auto')
 XGB = XGBClassifier(n_jobs=-1, random_state=42 )
learn.freeze_to(-1 )<train_model>
def train_accuracy(model):
model.fit(X_train, y_train)
 train_accuracy = model.score(X_train, y_train)
 train_accuracy = np.round(train_accuracy*100,2)
 return train_accuracy
learn.fit_one_cycle(5,3e-04 )<save_model>
train_accuracy = pd.DataFrame({'Training accuracy %':[train_accuracy(LR),train_accuracy(KNN),train_accuracy(DT),train_accuracy(RF),train_accuracy(SVM), train_accuracy(XGB)]})
 train_accuracy.index = ['Logistic Regression','KNN','Decision Tree','Random Forest','SVM','XGB']
sorted_train_accuracy = train_accuracy.sort_values(by='Training accuracy %', ascending = False)
 sorted_train_accuracy
learn.load('best_accuracy')
 learn.save('stage1' )<train_model>
def val_score(model):
val_score = cross_val_score(model, X_train, y_train, cv = 10, scoring = 'accuracy' ).mean()
val_score = np.round(val_score*100, 2)
 return val_score
val_score = pd.DataFrame({'val_score(%)':[val_score(LR), val_score(KNN), val_score(DT), val_score(RF), val_score(SVM), val_score(XGB)]})
 val_score.index = ['Logistic Regression', 'KNN','Decision Tree', 'Random Forest', 'SVC','XGB']
sorted_val_score = val_score.sort_values(by = 'val_score(%)', ascending = False)
 sorted_val_score
learn.fit_one_cycle(10, 7.59E-07 )<import_modules>
lr_params = {'penalty':['l1', 'l2'],
'C': np.logspace(0, 2, 4, 8 ,10)}
knn_params = {'n_neighbors':[4,5,6,7,8,9,10],
'weights':['uniform', 'distance'],
'algorithm':['auto', 'ball_tree','kd_tree','brute'],
'p':[1,2]}
dt_params = {'max_features': ['auto', 'sqrt', 'log2'],
'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
'min_samples_leaf':[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
'random_state':[46]}
rf_params = {'criterion':['gini','entropy'],
'n_estimators':[ 10, 30, 200, 400],
'min_samples_leaf':[1, 2, 3],
'min_samples_split':[3, 4, 6, 7],
'max_features':['sqrt', 'auto', 'log2'],
'random_state':[46]}
svc_params = {'C': [0.1, 1, 10,100],
'kernel': ['linear', 'rbf', 'poly', 'sigmoid'],
'gamma': [ 1, 0.1, 0.001, 0.0001]}
xgb_params = xgb_params_grid = {"learning_rate" : [0.05, 0.10, 0.15, 0.20, 0.25, 0.30 ] ,
"max_depth" : [ 3, 4, 5, 6, 8, 10, 12, 15],
"min_child_weight" : [ 1, 3, 5, 7 ],
"gamma" : [ 0.0, 0.1, 0.2 , 0.3, 0.4 ],
"colsample_bytree" : [ 0.3, 0.4, 0.5 , 0.7 ] }
import scipy as sp
from sklearn import metrics<compute_test_metric>
def tune_hyperparameters(model, param_grid):
global best_params, best_scores
grid = GridSearchCV(model,param_grid, verbose=0, cv=10, scoring='accuracy', n_jobs=-1)
 grid.fit(X_train, y_train)
 best_params, best_scores = grid.best_params_, np.round(grid.best_score_*100,2)
 return best_params, best_scores
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<load_pretrained>
tune_hyperparameters(LR, param_grid=lr_params)
 lr_best_params, lr_best_score = best_params,best_scores
print('Logistic Regression Best Score:', lr_best_score)
 print('And Best Parameters:', lr_best_params )
learn.load('best_accuracy');<predict_on_test>
tune_hyperparameters(KNN, param_grid=knn_params)
 knn_best_params, knn_best_score = best_params,best_scores
print('KNN Best Score:', knn_best_score)
 print('And Best Parameters:', knn_best_params )
valid_preds = learn.get_preds(ds_type=DatasetType.Valid )<train_model>
tune_hyperparameters(DT, param_grid=dt_params)
 dt_best_params, dt_best_score = best_params,best_scores
print('DT Best Score:', dt_best_score)
 print('And Best Parameters:', dt_best_params )
optR = OptimizedRounder()
optR.fit(valid_preds[0],valid_preds[1])
 coefficients = optR.coefficients()
coefficients=np.around(( coefficients), decimals=2)
 print(coefficients )<predict_on_test>
tune_hyperparameters(RF, param_grid=rf_params)
 rf_best_params, rf_best_score = best_params,best_scores
print('RF Best Score:', rf_best_score)
 print('And Best Parameters:', rf_best_params )
preds,y = learn.get_preds(DatasetType.Test )<save_to_csv>
tune_hyperparameters(XGB, param_grid=xgb_params)
 xgb_best_params, xgb_best_score = best_params,best_scores
print('SVC Best Score:', xgb_best_score)
 print('And Best Parameters:', xgb_best_params )
tst_pred = optR.predict(preds, coefficients)
 test_df.diagnosis = tst_pred.astype(int)
 test_df.to_csv('submission.csv',index=False )<count_values>
tune_hyperparameters(SVM, param_grid=svc_params)
 svc_best_params, svc_best_score = best_params,best_scores
print('SVC Best Score:', svc_best_score)
 print('And Best Parameters:', svc_best_params )
test_df['diagnosis'].value_counts()<define_variables>
lr = LogisticRegression(**lr_best_params)
 knn = KNeighborsClassifier(**knn_best_params)
 dt = DecisionTreeClassifier(**dt_best_params)
 rf = RandomForestClassifier(**rf_best_params)
 svc = SVC(**svc_best_params)
 xgb = XGBClassifier(**xgb_best_params )
data.show_batch()<import_modules>
models = {'LR':lr,'KNN':knn,'DT':dt,'RF':rf,'SVC':svc, 'XGB':xgb}
score = []
for x,(keys, items)in enumerate(models.items()):
items.fit(X_train,y_train)
 scores = cross_val_score(items, X_train, y_train, cv = 10, scoring='accuracy')*100
score.append(scores.mean())
 print('Mean Accuracy: %0.4f(+/- %0.4f)[%s]' %(scores.mean() , scores.std() , keys))
from fastai.vision import *<feature_engineering>
model_prediction = pd.DataFrame({'LR':lr.predict(X_test), 'KNN':knn.predict(X_test), 'DT':dt.predict(X_test),'RF':rf.predict(X_test), 'SVC':svc.predict(X_test), 'XGB':xgb.predict(X_test)})
 model_prediction.head()
vision.data.open_image=temp<set_options>
submisson = pd.DataFrame({"PassengerID":test["PassengerId"],"Survived":rf.predict(X_test)})
 submisson.to_csv('submisson_rf.csv',index=False )
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
submission = pd.DataFrame({'PassengerId':test['PassengerId'],'Survived':svc.predict(X_test)})
 submission.to_csv("submission_svc.csv", index = False )
<define_variables><EOS> 
submission = pd.DataFrame({'PassengerId':test['PassengerId'],'Survived':xgb.predict(X_test)})
 submission.to_csv("submission_xgb.csv", index = False )