Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<choose_model_class>
pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 500)
 df_train = pd.read_csv('.. /input/titanic/train.csv', delimiter = ",")
 df_test = pd.read_csv('.. /input/titanic/test.csv', delimiter = ",")
 df_sub = pd.read_csv('.. /input/titanic/gender_submission.csv', delimiter = "," )
def get_model_classif_nasnet_1() :
inputs = Input(( 96, 96, 3))
x1 = layers.Conv2D(32,3,padding='same' )(inputs)
 x1 = layers.BatchNormalization()(x1)
 x1 = layers.Activation('relu' )(x1)
 x1 = layers.Conv2D(32,3,padding='same' )(x1)
 x1 = layers.BatchNormalization()(x1)
 x1 = layers.Activation('relu' )(x1)
 x1 = layers.Conv2D(32,3,padding='same' )(x1)
 x1 = layers.BatchNormalization()(x1)
 x1 = layers.Activation('relu' )(x1)
 x1_s = layers.SeparableConv2D(32,3,padding='same' )(inputs)
 x1_s = layers.BatchNormalization()(x1_s)
 x1_s = layers.Activation('relu' )(x1_s)
 x1_s = layers.SeparableConv2D(32,3,padding='same' )(x1_s)
 x1_s = layers.BatchNormalization()(x1_s)
 x1_s = layers.Activation('relu' )(x1_s)
 x1_s = layers.SeparableConv2D(32,3,padding='same' )(x1_s)
 x1_s = layers.BatchNormalization()(x1_s)
 x1_s = layers.Activation('relu' )(x1_s)
 concetenated_0 = layers.concatenate([x1,x1_s])
 x2 = layers.Conv2D(64,3,padding='same' )(concetenated_0)
 x2 = layers.BatchNormalization()(x2)
 x2 = layers.Activation('relu' )(x2)
 x2 = layers.Conv2D(64,3,padding='same' )(x2)
 x2 = layers.BatchNormalization()(x2)
 x2 = layers.Activation('relu' )(x2)
 x2 = layers.Conv2D(64,3,padding='same' )(x2)
 x2 = layers.BatchNormalization()(x2)
 x2 = layers.Activation('relu' )(x2)
 residual_concetenated_0 = layers.Conv2D(64,1,strides=1,padding='same' )(concetenated_0)
 x2 = layers.add([x2,residual_concetenated_0])
 concetenates_x2_x1_s = layers.concatenate([x2,x1_s])
 x2 = layers.MaxPool2D(2,2 )(concetenates_x2_x1_s)
 x2_s = layers.SeparableConv2D(64,3,padding='same' )(x2)
 x2_s = layers.BatchNormalization()(x2_s)
 x2_s = layers.Activation('relu' )(x2_s)
 x2_s = layers.SeparableConv2D(64,3,padding='same' )(x2_s)
 x2_s= layers.BatchNormalization()(x2_s)
 x2_s= layers.Activation('relu' )(x2_s)
 x2_s = layers.SeparableConv2D(64,3,padding='same' )(x2_s)
 x2_s= layers.BatchNormalization()(x2_s)
 x2_s= layers.Activation('relu' )(x2_s)
 x2_s = layers.Conv2D(96,1,strides=1,padding='same' )(x2_s)
 x2_s = layers.add([x2_s,x2])
 x2_s = layers.MaxPool2D(2,2 )(x2_s)
 x3 = layers.Conv2D(128,3,padding='same' )(x2)
 x3 = layers.BatchNormalization()(x3)
 x3 = layers.Activation('relu' )(x3)
 x3 = layers.Conv2D(128,3,padding='same' )(x3)
 x3 = layers.BatchNormalization()(x3)
 x3 = layers.Activation('relu' )(x3)
 x3 = layers.Conv2D(128,3,padding='same' )(x3)
 x3 = layers.BatchNormalization()(x3)
 x3 = layers.Activation('relu' )(x3)
 residual_x2 = layers.Conv2D(128,1,strides=1,padding='same' )(x2)
 x3 = layers.add([residual_x2,x3])
 x3_x3 = layers.Conv2D(128,3,padding='same' )(x3)
 x3_x3 = layers.BatchNormalization()(x3_x3)
 x3_x3 = layers.Activation('relu' )(x3_x3)
 x3_x3 = layers.Conv2D(128,3,padding='same' )(x3_x3)
 x3_x3 = layers.BatchNormalization()(x3_x3)
 x3_x3 = layers.Activation('relu' )(x3_x3)
 x3_x3 = layers.Conv2D(128,3,padding='same' )(x3_x3)
 x3_x3 = layers.BatchNormalization()(x3_x3)
 x3_x3 = layers.Activation('relu' )(x3_x3)
 x3_x3 = layers.add([x3,x3_x3])
 x3_x3 = layers.MaxPool2D(2,2 )(x3_x3)
 concetenated_1 = layers.concatenate([x3_x3,x2_s])
 x3_s = layers.SeparableConv2D(128,3,padding='same' )(concetenated_1)
 x3_s = layers.BatchNormalization()(x3_s)
 x3_s = layers.Activation('relu' )(x3_s)
 x3_s = layers.SeparableConv2D(128,3,padding='same' )(x3_s)
 x3_s= layers.BatchNormalization()(x3_s)
 x3_s= layers.Activation('relu' )(x3_s)
 x3_s = layers.SeparableConv2D(128,3,padding='same' )(x3_s)
 x3_s= layers.BatchNormalization()(x3_s)
 x3_s= layers.Activation('relu' )(x3_s)
 x3_s = layers.add([x3_s,x3_x3])
 x3_s = layers.MaxPool2D(2,2 )(x3_s)
 x4 = layers.Conv2D(256,3,padding='same' )(x3_x3)
 x4 = layers.BatchNormalization()(x4)
 x4 = layers.Activation('relu' )(x4)
 x4 = layers.Conv2D(256,3,padding='same' )(x4)
 x4 = layers.BatchNormalization()(x4)
 x4 = layers.Activation('relu' )(x4)
 x4 = layers.Conv2D(256,3,padding='same' )(x4)
 x4 = layers.BatchNormalization()(x4)
 x4 = layers.Activation('relu' )(x4)
 residual_x3 = layers.Conv2D(256,1,strides=1,padding='same' )(x3_x3)
 x4 = layers.add([residual_x3,x4])
 x4_x4 = layers.Conv2D(256,3,padding='same' )(x4)
 x4_x4 = layers.BatchNormalization()(x4_x4)
 x4_x4 = layers.Activation('relu' )(x4_x4)
 x4_x4 = layers.Conv2D(256,3,padding='same' )(x4_x4)
 x4_x4 = layers.BatchNormalization()(x4_x4)
 x4_x4 = layers.Activation('relu' )(x4_x4)
 x4_x4 = layers.Conv2D(256,3,padding='same' )(x4_x4)
 x4_x4 = layers.BatchNormalization()(x4_x4)
 x4_x4 = layers.Activation('relu' )(x4_x4)
 x4_x4 = layers.add([x4,x4_x4])
 x4_x4 = layers.MaxPool2D(2,2 )(x4_x4)
 concetenated_2 = layers.concatenate([x4_x4,x3_s])
 x4_s = layers.SeparableConv2D(256,3,padding='same' )(concetenated_2)
 x4_s = layers.BatchNormalization()(x4_s)
 x4_s = layers.Activation('relu' )(x4_s)
 x4_s = layers.SeparableConv2D(256,3,padding='same' )(x4_s)
 x4_s= layers.BatchNormalization()(x4_s)
 x4_s= layers.Activation('relu' )(x4_s)
 x4_s = layers.SeparableConv2D(256,3,padding='same' )(x4_s)
 x4_s= layers.BatchNormalization()(x4_s)
 x4_s= layers.Activation('relu' )(x4_s)
 x4_s = layers.add([x4_s,x4_x4])
 x4_s = layers.MaxPool2D(2,2 )(x4_s)
 x5 = layers.Conv2D(512,3,padding='same' )(x4_x4)
 x5 = layers.BatchNormalization()(x5)
 x5 = layers.Activation('relu' )(x5)
 x5 = layers.Conv2D(512,3,padding='same' )(x5)
 x5 = layers.BatchNormalization()(x5)
 x5 = layers.Activation('relu' )(x5)
 x5 = layers.Conv2D(512,3,padding='same' )(x5)
 x5 = layers.BatchNormalization()(x5)
 x5 = layers.Activation('relu' )(x5)
 residual_x4 = layers.Conv2D(512,1,strides=1,padding='same' )(x4_x4)
 x5 = layers.add([residual_x4,x5])
 x5_x5 = layers.Conv2D(512,3,padding='same' )(x5)
 x5_x5 = layers.BatchNormalization()(x5_x5)
 x5_x5 = layers.Activation('relu' )(x5_x5)
 x5_x5 = layers.Conv2D(512,3,padding='same' )(x5_x5)
 x5_x5 = layers.BatchNormalization()(x5_x5)
 x5_x5 = layers.Activation('relu' )(x5_x5)
 x5_x5 = layers.Conv2D(512,3,padding='same' )(x5_x5)
 x5_x5 = layers.BatchNormalization()(x5_x5)
 x5_x5 = layers.Activation('relu' )(x5_x5)
 x5_x5 = layers.add([x5,x5_x5])
 x5_x5 = layers.MaxPool2D(2,2 )(x5_x5)
 concetenated_3 = layers.concatenate([x5_x5,x4_s])
 x5_s = layers.SeparableConv2D(512,3,padding='same' )(concetenated_3)
 x5_s = layers.BatchNormalization()(x5_s)
 x5_s = layers.Activation('relu' )(x5_s)
 x5_s = layers.SeparableConv2D(512,3,padding='same' )(x5_s)
 x5_s= layers.BatchNormalization()(x5_s)
 x5_s= layers.Activation('relu' )(x5_s)
 x5_s = layers.SeparableConv2D(512,3,padding='same' )(x5_s)
 x5_s= layers.BatchNormalization()(x5_s)
 x5_s= layers.Activation('relu' )(x5_s)
 x5_s = layers.add([x5_s,x5_x5])
 x = layers.GlobalAveragePooling2D()(x5_s)
 x = layers.Dense(64 )(x)
 x = layers.BatchNormalization()(x)
 x = layers.Activation('relu' )(x)
 x = layers.Dropout(0.3 )(x)
 output_tensor = layers.Dense(1,activation='sigmoid' )(x)
 model = Model(inputs,output_tensor)
 model.compile(optimizer=Adam(0.001), loss='binary_crossentropy', metrics=['acc'])
 model.summary()
return model<find_best_params>
df = pd.concat([df_train, df_test],sort=False)
 df.reset_index(drop=True,inplace=True )
model = get_model_classif_nasnet_1() 
<train_model>
age60=df[(df['Age']>60)&(df['Age']<70)]
age60.groupby(['Pclass','Embarked','SibSp','Parch'] ).median()
df.loc[df['PassengerId'] == 1044, 'Fare'] = 7.9
model.load_weights('.. /input/model-3/model_3.h5')
 batch_size=128
h5_path = "model_4.h5"
checkpoint = ModelCheckpoint(h5_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
 history = model.fit_generator(
 data_gen(train, id_label_map, batch_size, augment=True),
validation_data=data_gen(val, id_label_map, batch_size),
epochs=5, verbose=1,
callbacks=[checkpoint],
steps_per_epoch=len(train)// batch_size,
validation_steps=len(val)// batch_size)
<define_variables>
df['Fare']=np.round(df['Fare'])
 p01 = df['Fare'].quantile(0.01)
 p99 = df['Fare'].quantile(0.99)
 df['Fare'] = df['Fare'].clip(p01, p99 )
preds = []
ids = []<predict_on_test>
df['Farecut']=pd.cut(df['Fare'], 5, labels=False )
model.load_weights(h5_path)
 for batch in chunker(test_files, batch_size):
X = [preprocess_input(cv2.imread(x)) for x in batch]
ids_batch = [get_id_from_file_path(x)for x in batch]
X = np.array(X)
 preds_batch =(( model.predict(X ).ravel() *model.predict(X[:, ::-1, :, :] ).ravel() *model.predict(X[:, ::-1, ::-1, :] ).ravel() *model.predict(X[:, :, ::-1, :] ).ravel())**0.25 ).tolist()
preds += preds_batch
ids += ids_batch<save_to_csv>
column = 'mrms'
df[column] = df['Name'].str.extract('([A-Za-z]+)\.', expand = False)
 df[column] = df[column].replace(['Capt', 'Countess','Lady','Sir','Jonkheer', 'Don','Dona', 'Mlle','Mme','Major'], 'test_non')
 df[column] = df[column].replace(['Ms'], 'Miss')
 df[column] = df[column].replace(['Col','Dr', 'Rev'], 'Other' )
df = pd.DataFrame({'id':ids, 'label':preds})
 df.to_csv("baseline_nasnet.csv", index=False)
 df.head()<import_modules>
column='Embarked'
df['Embarked'].fillna('S',inplace=True )
plt.rcParams['figure.figsize'] =(20,10)
 def append_ext(fn):
return fn+".tif"
print(os.listdir(".. /input")) 
<set_options>
df.loc[df['Parch'] >= 3, 'Parch'] = 3
df.loc[df['SibSp'] >= 3, 'SibSp'] = 3
seed(1)
 set_random_seed(1 )<feature_engineering>
df['Parch+SibSp']=df['Parch']+df['SibSp']
traindf=pd.read_csv(".. /input/train_labels.csv",dtype=str)
 train_size = 180000
traindf = traindf.sort_values(by=['label','id'])
 traindf = traindf.iloc[:int(train_size/2)].append(traindf.iloc[-int(train_size/2):])
 testdf=pd.read_csv(".. /input/sample_submission.csv",dtype=str)
 traindf["id"]=traindf["id"].apply(append_ext)
 testdf["id"]=testdf["id"].apply(append_ext)
 datagen=ImageDataGenerator(rescale=1./255.,validation_split=0.25 )<create_dataframe>
df['Parch-SibSp']=df['Parch']-df['SibSp']
batch_size = 64
train_generator=datagen.flow_from_dataframe(
 dataframe=traindf,
directory=".. /input/train/",
x_col="id",
y_col="label",
subset="training",
batch_size=batch_size,
seed=42,
shuffle=True,
class_mode="binary",
target_size=(96, 96)
)
 valid_generator=datagen.flow_from_dataframe(
 dataframe=traindf,
directory=".. /input/train/",
x_col="id",
y_col="label",
subset="validation",
batch_size=batch_size,
seed=42,
shuffle=True,
class_mode="binary",
target_size=(96, 96)
)
 test_datagen=ImageDataGenerator(rescale=1./255.)
test_generator=test_datagen.flow_from_dataframe(
 dataframe=testdf,
directory=".. /input/test/",
x_col="id",
y_col=None,
batch_size=batch_size,
seed=42,
shuffle=False,
class_mode=None,
target_size=(96, 96)
 )<compute_test_metric>
l=list(['Pclass','SibSp','Fare','Parch'])
 df['Agemedian'] = df['Age'].fillna(df.groupby(l)['Age'].transform('median'))
df['Agemedian'] = df['Agemedian'].fillna(df['Agemedian'].median())
 df['Agemedian'] = np.round(df['Agemedian'] )
def auc(y_true, y_pred):
auc = tf.metrics.auc(y_true, y_pred)[1]
K.get_session().run(tf.local_variables_initializer())
 return auc 
<choose_model_class>
column = 'Cabin'
df[column+'head'] = df[column].str.extract('([A-Za-z]+)', expand = False)
 labels, uniques = pd.factorize(df['Cabinhead'])
 df['Cabinhead']=labels
df['Cabinhead'].fillna(0,inplace=True)
 df['Cabinhead'].replace(-1,np.NaN,inplace=True)
 Ticket = df['Ticket'].str.extract(' (.*)\s (.*)')
 df['Ticket_head']=Ticket[0]
df['Ticket_num']=Ticket[1]
l = list(['Pclass','Farecut'])
 df['Cabinheadmedian'] = df['Cabinhead'].fillna(df.groupby(l)['Cabinhead'].transform('median'))
df['Cabinheadmedian'] = np.round(df['Cabinheadmedian'] )
def make_model(model_choice, input_tensor):

 base_model = model_choice
x = base_model(input_tensor)
 out = Dense(1, activation="softmax" )(x)
 model = Model(input_tensor, out)
 model.summary()
return model<choose_model_class>
l = list(['Farecut'])
 df['Cabinheadmedian'] = df['Cabinhead'].fillna(df.groupby(l)['Cabinhead'].transform('median'))
input_tensor = Input(( 96, 96, 3))
VGG19_model=make_model(VGG19(include_top=False, pooling='avg', input_tensor=input_tensor), input_tensor )<choose_model_class>
for column in ['mrms','Sex','Embarked','Ticket_head']:
labels, uniques = pd.factorize(df[column])
 df[column]=labels
best_model_weights = './base.model'
checkpoint = ModelCheckpoint(
 best_model_weights,
monitor = 'val_loss',
save_best_only = True,
mode='min',
verbose=1,
save_weights_only=False,
period = 1)
 earlyStop = EarlyStopping(
 monitor = 'val_loss',
min_delta=0.01,
patience = 5,
verbose = 1,
mode = 'min')
 tensorBoard = TensorBoard(
 log_dir = './logs',
histogram_freq=0,
batch_size=batch_size,
write_graph=True,
write_grads=True,
write_images=False)
 reduce = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=3,
verbose=1,
mode='min' 
)
 csvlogger = CSVLogger(filename='training_csv.log',
separator=',',
append=False)
 callbacks = [checkpoint, tensorBoard, csvlogger, reduce]<define_variables>
df = df.drop(['Name','Ticket','Cabin','Cabinhead','Age','Fare','Ticket_num'], axis=1 )
start=datetime.now()
STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size
STEP_SIZE_VALID=valid_generator.n//valid_generator.batch_size
STEP_SIZE_TEST=test_generator.n//test_generator.batch_size<train_model>
import operator
import math
import random
from deap import gp
from deap import algorithms
from deap import base
from deap import creator
from deap import tools
VGG19_model.compile(optimizer=SGD(lr=1e-4, momentum=0.99), loss=binary_crossentropy, metrics=['accuracy', auc])
 history = VGG19_model.fit_generator(generator=train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
epochs=6,
callbacks=callbacks 
)
 end = datetime.now()
print("Elapsed time:",end-start )<load_pretrained>
train=df
VGG19_model.load_weights(best_model_weights)
 VGG19_model.evaluate_generator(generator=valid_generator,
steps=STEP_SIZE_VALID )<predict_on_test>
target = train['Survived']
def get_results(model_used):
test_generator.reset()
pred=model_used.predict_generator(test_generator,
steps=STEP_SIZE_TEST+1,
verbose=1)
 predicted_class_indices=np.argmax(pred,axis=1)
 labels =(train_generator.class_indices)
 labels = dict(( v,k)for k,v in labels.items())
 predictions = [labels[k] for k in predicted_class_indices]
filenames=test_generator.filenames
results=pd.DataFrame({"id":[f[:-4] for f in filenames],
"label":[x[0] for x in pred]})
 return results<find_best_params>
def mydeap(mungedtrain):
inputs = mungedtrain.iloc[:,2:14].values.tolist()
outputs = mungedtrain['Survived'].values.tolist()
def protectedDiv(left, right):
try:
return left / right
except ZeroDivisionError:
return 1
pset = gp.PrimitiveSet("MAIN", 12)
 pset.addPrimitive(operator.add, 2)
 pset.addPrimitive(operator.sub, 2)
 pset.addPrimitive(operator.mul, 2)
 pset.addPrimitive(protectedDiv, 2)
 pset.addPrimitive(operator.neg, 1)
 pset.addPrimitive(math.cos, 1)
 pset.addPrimitive(math.sin, 1)
 pset.addPrimitive(max, 2)
 pset.addPrimitive(min, 2)
 pset.addEphemeralConstant("rand101", lambda: random.uniform(-10,10))
pset.renameArguments(ARG0='x1')
 pset.renameArguments(ARG1='x2')
 pset.renameArguments(ARG2='x3')
 pset.renameArguments(ARG3='x4')
 pset.renameArguments(ARG4='x5')
 pset.renameArguments(ARG5='x6')
 pset.renameArguments(ARG6='x7')
 pset.renameArguments(ARG7='x8')
 pset.renameArguments(ARG8='x9')
 pset.renameArguments(ARG9='x10')
 pset.renameArguments(ARG10='x11')
 pset.renameArguments(ARG11='x12')
 creator.create("FitnessMin", base.Fitness, weights=(1.0,))
creator.create("Individual", gp.PrimitiveTree, fitness=creator.FitnessMin)
 toolbox = base.Toolbox()
toolbox.register("expr", gp.genHalfAndHalf, pset=pset, min_=1, max_=3)
 toolbox.register("individual", tools.initIterate, creator.Individual, toolbox.expr)
 toolbox.register("population", tools.initRepeat, list, toolbox.individual)
 toolbox.register("compile", gp.compile, pset=pset)
 def evalSymbReg(individual):
func = toolbox.compile(expr=individual)
 return sum(round(1.-(1./(1.+numpy.exp(-func(*in_)))))== out for in_, out in zip(inputs, outputs)) /len(mungedtrain),
toolbox.register("evaluate", evalSymbReg)
 toolbox.register("select", tools.selTournament, tournsize=3)
 toolbox.register("mate", gp.cxOnePoint)
 toolbox.register("expr_mut", gp.genFull, min_=0, max_=2)
 toolbox.register("mutate", gp.mutUniform, expr=toolbox.expr_mut, pset=pset)
 toolbox.decorate("mate", gp.staticLimit(key=operator.attrgetter("height"), max_value=17))
toolbox.decorate("mutate", gp.staticLimit(key=operator.attrgetter("height"), max_value=17))
random.seed(318)
 pop = toolbox.population(n=300)
 hof = tools.HallOfFame(1)
 stats_fit = tools.Statistics(lambda ind: ind.fitness.values)
 stats_size = tools.Statistics(len)
 mstats = tools.MultiStatistics(fitness=stats_fit, size=stats_size)
 mstats.register("avg", numpy.mean)
 mstats.register("std", numpy.std)
 mstats.register("min", numpy.min)
 mstats.register("max", numpy.max)
 pop, log = algorithms.eaSimple(pop, toolbox, 0.5, 0.2, 1000, stats=mstats,
halloffame=hof, verbose=True)
 print(hof[0])
 func2 =toolbox.compile(expr=hof[0])
 return func2
results = get_results(VGG19_model )<save_to_csv>
if __name__ == "__main__":
mungedtrain = train[:df_train.shape[0]]
GeneticFunction = mydeap(mungedtrain)
 mytrain = mungedtrain.iloc[:,2:14].values.tolist()
trainPredictions = Outputs(np.array([GeneticFunction(*x)for x in mytrain]))
pdtrain = pd.DataFrame({'PassengerId': mungedtrain.PassengerId.astype(int),
'Predicted': trainPredictions.astype(int),
'Survived': mungedtrain.Survived.astype(int)})
 pdtrain.to_csv('MYgptrain.csv', index=False )
results.to_csv("results.csv",
index=False )<set_options>
print(accuracy_score(df_train.Survived.astype(int),trainPredictions.astype(int)))
 mungedtest = train[df_train.shape[0]:]
mytest = mungedtest.iloc[:,2:14].values.tolist()
testPredictions = Outputs(np.array([GeneticFunction(*x)for x in mytest]))
pdtest = pd.DataFrame({'PassengerId': mungedtest.PassengerId.astype(int),
'Survived': testPredictions.astype(int)})
 pdtest.to_csv('gptest.csv', index=False )
<normalization><EOS> 
print('end' )