Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<prepare_output>
sns.set()
%matplotlib inline
predictions = []
for data, target in testloader:
if train_on_gpu:
data, target = data.cuda() , target.cuda()
output = model(data)
 _, pred = torch.max(output, 1)
 for i in range(len(pred)) :
predictions.append(int(pred[i]))
submission['Category'] = predictions 
<save_to_csv>
train_data = pd.read_csv('train_prep.csv')
 test_data = pd.read_csv('test_prep.csv')
 train_data.columns
submission.to_csv('submission.csv', index=False, encoding='utf-8')
<train_model>
train_data.set_index('PassengerId',inplace = True)
 test_data.set_index('PassengerId',inplace = True )
X_train = np.load(input_dir + 'X_train.npy')
 X_test = np.load(input_dir + 'X_test.npy')
 y_train = np.load(input_dir + 'y_train.npy')
 print("X_train shape :",X_train.shape)
 print("X_test shape :",X_test.shape)
 print("y_train shape :",y_train.shape )<save_to_csv>
all_data = pd.concat([train_data,test_data] )
pred = pd.DataFrame()
pred['Id'] = np.arange(X_test.shape[0])
 pred['Prediction'] = np.zeros(X_test.shape[0],dtype = np.int)
 pred.to_csv('pred.csv',index=False)
 pred.head()<set_options>
label = LabelEncoder()
warnings.filterwarnings('ignore')
 np.seed = 1324
sns.set(style="white")
 %matplotlib inline 
<import_modules>
all_data['FareBin_Code'] = label.fit_transform(all_data['FareBin'])
 all_data['AgeBin_Code'] = label.fit_transform(all_data['AgeBin'] )
import cv2
import sys
import os
import keras<import_modules>
all_data.drop(columns=['AgeBin'],inplace=True)
 all_data.drop(columns=['FareBin'],inplace=True )
from keras.models import Sequential
from keras import layers
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense
from keras.layers import SpatialDropout2D
from keras.layers import Dropout
from keras.layers import Activation, BatchNormalization
from keras.optimizers import SGD
from keras import regularizers
from keras.callbacks import TerminateOnNaN, ModelCheckpoint, LearningRateScheduler
from keras.utils.np_utils import to_categorical
import tensorflow as tf
from PIL import Image, ImageEnhance
from keras.utils import np_utils
from keras.preprocessing.image import ImageDataGenerator 
<save_to_csv>
for i in range(1,len(all_data)+1):
all_data.loc[i,'Srate']=0.5
def submit_file(y_pred, filename):
submit = pd.Series(y_pred.reshape(y_pred.shape[0]), name='target' ).replace({0: 'Bird', 1: 'Airplane'})
 submit.to_csv(filename +'.csv', index_label='id', header=True)
 print('file created' )<categorify>
for i in range(1,len(all_data)+1):
number = all_data.loc[i,'TicketNumber']
group = all_data.loc[i,'TicketGroup']
if group>1:
max = all_data.drop(i)[all_data.drop(i)['TicketNumber']==number]['Survived'].max()
if not pd.isna(max):
all_data.loc[i,'Srate']=all_data.drop(i)[all_data.drop(i)['TicketNumber']==number]['Survived'].max()
def flip_image(array, show=False):

 image_obj = Image.fromarray(array.astype('uint8'))
rotated_image = image_obj.transpose(Image.FLIP_LEFT_RIGHT)
 flipped_array = np.array(rotated_image, dtype=np.int)
 if show:
plt.imshow(flipped_array)
 return flipped_array
def contrast_image(array, c_level=4.0, show=False):

 image_obj = Image.fromarray(array.astype('uint8'))
enhancer = ImageEnhance.Contrast(image_obj)
 enhanced_im = enhancer.enhance(c_level)
 contrasted_array = np.array(enhanced_im, dtype=np.int)
 if show:
plt.imshow(contrasted_array)
 return contrasted_array<load_from_csv>
train_data = all_data[:891]
test_data = all_data[891:]
train_x = pd.read_csv('.. /input/train_x.csv', index_col=0, header=None)
 train_y = pd.read_csv('.. /input/train_y.csv', index_col=0)
 test_x = pd.read_csv('.. /input/test_x.csv', index_col=0, header=None )<categorify>
numerical = ["Pclass","Age","SibSp","Parch","Fare","TicketGroup","FamilySize","FareBin_Code","AgeBin_Code","Srate","TicketNumber"]
categorical = ["Embarked","Title","Sex"]
all_features = numerical + categorical
train_y.replace(['Bird', 'Airplane'], [0,1], inplace=True)
 X = train_x.values.reshape(7200, 32, 32, 3)
 X_test = test_x.values.reshape(4800, 32, 32, 3)
 y_train = train_y.values 
<compute_test_metric>
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_val_score
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import OrdinalEncoder
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import VotingClassifier
viz_img(X[100], y_train[100] )<compute_test_metric>
y = train_data['Survived']
X = train_data.drop(columns = ['Survived'] )
viz_img(X[452], y_train[452] )<split>
log_reg = LogisticRegression(C=1.9 )
X = X.astype('float32')
 X_test = X_test.astype('float32')
 x_train, x_test, y_train, y_test = train_test_split(X, y_train, test_size=0.3, random_state=12000 )<compute_test_metric>
num_att_LR = ["Pclass","Age","TicketNumber","SibSp","Parch","Srate","Fare"]
cat_att1_LR = ["Sex"]
cat_att2_LR = ["Title"]
drop_att_LR = list(set(all_features)- set(num_att_LR+cat_att1_LR+cat_att2_LR))
full_pipline_LR = ColumnTransformer([ 
("toDrop",'drop',drop_att_LR), 
("numerical",StandardScaler() ,num_att_LR), 
("categorical1",OrdinalEncoder() ,cat_att1_LR), 
("categorical2",OneHotEncoder() ,cat_att2_LR)
 ] )
mean = np.mean(x_train, axis=(0,1,2,3))
std = np.std(x_train,axis=(0,1,2,3))
x_train =(x_train-mean)/(std + 1e-7)
 x_test =(x_test-mean)/(std + 1e-7)
 X_test =(X_test-mean)/(std + 1e-7 )<choose_model_class>
X_prep = full_pipline_LR.fit_transform(X)
 cross_val_score(log_reg,X_prep,y,cv=10,scoring='roc_auc' ).mean()
weight_decay = 1e-4
model = Sequential()
model.add(Conv2D(32,(3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay),
input_shape=x_train.shape[1:]))
model.add(Activation('elu'))
model.add(BatchNormalization())
 model.add(Conv2D(32,(3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
 model.add(Activation('elu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.2))
model.add(Conv2D(64,(3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
 model.add(Activation('elu'))
model.add(BatchNormalization())
 model.add(Conv2D(64,(3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
 model.add(Activation('elu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.3))
model.add(Conv2D(128,(3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
 model.add(Activation('elu'))
model.add(BatchNormalization())
 model.add(Conv2D(128,(3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
 model.add(Activation('elu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(1, activation='sigmoid'))
model.summary()<train_model>
cat_bayes = CategoricalNB(alpha=0.05 )
datagen = ImageDataGenerator(
 rotation_range=15,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True, 
)
 datagen.fit(x_train )<train_model>
num_att_NB = ["FareBin_Code","AgeBin_Code"]
cat_att_NB = ["Title"]
drop_att_NB = list(set(all_features)- set(num_att_NB+cat_att_NB))
full_pipline_NB = ColumnTransformer([ 
("cat",OrdinalEncoder() ,cat_att_NB), 
("num",'passthrough',num_att_NB), 
("drop",'drop',drop_att_NB)
 ] )
batch_size = 64
opt_rms = keras.optimizers.rmsprop(lr=0.001,decay=1e-6)
 model.compile(loss='binary_crossentropy', optimizer=opt_rms, metrics=['accuracy'])
 model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch=x_train.shape[0] // batch_size, epochs=125,
verbose=1,validation_data=(x_test,y_test),callbacks=[LearningRateScheduler(lr_schedule)] )<predict_on_test>
X_prep = full_pipline_NB.fit_transform(X)
 cross_val_score(cat_bayes,X_prep,y,cv=10,scoring='roc_auc' ).mean()
y_pred = model.predict_classes(X_test )<set_options>
from sklearn import tree
warnings.filterwarnings('ignore')
 sns.set()
%config InlineBackend.figure_format = 'svg'
print(os.listdir('.. /input/'))<load_from_csv>
num_att_DT =["Pclass","FamilySize","Srate","AgeBin_Code","FareBin_Code"]
cat_att_DT = ["Sex"]
drop_att_DT = list(set(all_features)- set(num_att_DT + cat_att_DT))
full_pipline_DT = ColumnTransformer([ 
("droping",'drop',drop_att_DT), 
("num",'passthrough',num_att_DT), 
("cat",OrdinalEncoder() ,cat_att_DT),
] )
df = pd.read_csv('.. /input/train.csv')
 df.head()<load_from_csv>
X_prep = full_pipline_DT.fit_transform(X )
test_df = pd.read_csv('.. /input/test.csv' )<feature_engineering>
criterion = ["gini", "entropy"]
max_depth = list(range(1,12,1))
max_features = [1,2,3,4,5,6]
splitter = ["best","random"]
min_samples_split = [2,3]
hyperparams = {'criterion': criterion, 'max_depth': max_depth, 'max_features': max_features,
'splitter': splitter, 'min_samples_split': min_samples_split}
gd = GridSearchCV(estimator = tree.DecisionTreeClassifier(random_state=1), param_grid = hyperparams, verbose=0,
cv=10, scoring = "roc_auc" )
df['char_count'] = df['review'].apply(len)
 df['word_count'] = df['review'].apply(lambda x: len(x.split()))
df['word_density'] = df['char_count'] /(df['word_count']+1)
 df['punctuation_count'] = df['review'].apply(lambda x: len("".join(_ for _ in x if _ in string.punctuation)) )<split>
gd.fit(X_prep, y)
 print(gd.best_score_)
 print(gd.best_estimator_ )
X_train, X_val, y_train, y_val = train_test_split(df['review'], df['sentiment'], test_size=0.3,
random_state=17 )<feature_engineering>
DT_clf = tree.DecisionTreeClassifier(criterion='entropy', max_depth=5, max_features=6,
random_state=1, splitter='random' )
tfidf_vect = TfidfVectorizer(analyzer='word', token_pattern=r'\w{1,}', max_features=5000)
 tfidf_vect.fit(df['review'])
 xtrain_tfidf = tfidf_vect.transform(X_train)
 xvalid_tfidf = tfidf_vect.transform(X_val)
 tfidf_vect_ngram = TfidfVectorizer(analyzer='word', token_pattern=r'\w{1,}', ngram_range=(2,3), max_features=5000)
 tfidf_vect_ngram.fit(df['review'])
 xtrain_tfidf_ngram = tfidf_vect_ngram.transform(X_train)
 xvalid_tfidf_ngram = tfidf_vect_ngram.transform(X_val)
 tfidf_vect_ngram_chars = TfidfVectorizer(analyzer='char', token_pattern=r'\w{1,}', ngram_range=(2,3), max_features=5000)
 tfidf_vect_ngram_chars.fit(df['review'])
 xtrain_tfidf_ngram_chars = tfidf_vect_ngram_chars.transform(X_train)
 xvalid_tfidf_ngram_chars = tfidf_vect_ngram_chars.transform(X_val)
 xtest_tfidf = tfidf_vect.transform(test_df['review'])
 xtest_tfidf_ngram = tfidf_vect_ngram.transform(test_df['review'])
 xtest_tfidf_ngram_chars = tfidf_vect_ngram_chars.transform(test_df['review'] )<feature_engineering>
svm_clf = LinearSVC(C=0.1, loss = 'hinge', max_iter=100000, random_state=1, )
count_vect = CountVectorizer(analyzer='word', token_pattern=r'\w{1,}')
 count_vect.fit(df['review'])
 xtrain_count = count_vect.transform(X_train)
 xvalid_count = count_vect.transform(X_val)
 xtest_count = count_vect.transform(test_df['review'] )<choose_model_class>
num_att_SVM = numerical
cat_att1_SVM = ["Sex"]
cat_att2_SVM = ["Title"]
drop_att_SVM = list(set(all_features)- set(num_att_SVM + cat_att1_SVM+cat_att2_SVM))
full_pipline_SVM = ColumnTransformer([ 
("drop",'drop',drop_att_SVM), 
("cat",OrdinalEncoder() ,cat_att1_SVM), 
("num",StandardScaler() ,num_att_SVM), 
("cat2",OneHotEncoder() ,cat_att2_SVM),
] )
model1 = linear_model.LogisticRegression()
model1.fit(xtrain_count, y_train)
 accuracy=model1.score(xvalid_count, y_val)
 print('Accuracy Count LR:', accuracy)
 test_pred1=model1.predict(xtest_count)
 model2 = linear_model.LogisticRegression()
model2.fit(xtrain_tfidf, y_train)
 accuracy=model2.score(xvalid_tfidf, y_val)
 print('Accuracy TFIDF LR:', accuracy)
 test_pred2=model2.predict(xtest_tfidf)
 model3 = linear_model.LogisticRegression()
model3.fit(xtrain_tfidf_ngram, y_train)
 accuracy = model3.score(xvalid_tfidf_ngram, y_val)
 print('Accuracy TFIDF NGRAM LR:', accuracy)
 test_pred3 = model3.predict(xtest_tfidf_ngram )<concatenate>
X_prep = full_pipline_SVM.fit_transform(X)
 cross_val_score(svm_clf,X_prep,y,cv=10,scoring='roc_auc' ).mean()
final_pred = np.array([])
 for i in range(0,len(test_df['review'])) :
final_pred = np.append(final_pred, np.argmax(np.bincount([test_pred1[i], test_pred2[i], test_pred3[i]])) )<create_dataframe>
LR_pipe = Pipeline([('logReg pipline', full_pipline_LR), 
('log_reg', log_reg)])
 cross_val_score(LR_pipe,X,y,cv=10,scoring='accuracy' ).mean()
sub_df = pd.DataFrame()
sub_df['Id'] = test_df['Id']
sub_df['sentiment'] = [int(i)for i in final_pred]<save_to_csv>
NB_pipe = Pipeline([('NaiveBayes pipline', full_pipline_NB), 
('cat_bayes', cat_bayes)])
 cross_val_score(NB_pipe,X,y,cv=10,scoring='accuracy' ).mean()
sub_df.to_csv('my_submission.csv', index=False )<import_modules>
DT_pipe = Pipeline([('DecisionTree pipline', full_pipline_DT), 
('Decision Tree', DT_clf)])
 cross_val_score(DT_pipe,X,y,cv=10,scoring='accuracy' ).mean()
random_seed = 1213
np.random.seed(random_seed)
<load_from_csv>
SVM_pipe = Pipeline([('SVM pipline', full_pipline_SVM), 
('SVM', svm_clf)])
 cross_val_score(SVM_pipe,X,y,cv=10,scoring='accuracy' ).mean()
train = pd.read_csv('.. /input/ciproject/persianMNIST_train.csv')
 test = pd.read_csv('.. /input/ciproject/persianMNIST_test.csv')
 Y = train['0']
X = train.drop(labels="0", axis=1)
 X = X.values.reshape(-1, 28, 28, 1)
 test = test.values.reshape(-1, 28, 28, 1)
 X_train, X_val, Y_train, Y_val = train_test_split(X, Y, test_size=0.10, random_state=random_seed)
 print(X.shape, test.shape )<choose_model_class>
voting_clf_4 = VotingClassifier(
 estimators = [('Logistic Regresion',LR_pipe),('SVM',SVM_pipe),('Naive Bayes',NB_pipe),('Decision Tree',DT_pipe)],
voting = 'hard' 
)
 cross_val_score(voting_clf_4,X,y,cv=10,scoring='accuracy' ).mean()
learning_rate_reduction = ReduceLROnPlateau(monitor = 'val_acc',
patience = 3,
verbose = 1,
factor = 0.5,
min_lr = 0.0001)
 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=15)
 def new_model(hidden=512, learning_rate=0.00128):
INPUT = Input(( 28, 28, 1))
inputs = Conv2D(64,(7, 7), activation='relu', padding='same' )(INPUT)
 inputs = MaxPool2D(pool_size=(5,5), strides=(2,2))(inputs)
 inputs = BatchNormalization()(inputs)
 inputs = Activation('relu' )(inputs)
 inputs = Dropout(0.5 )(inputs)
 tower_1 = Conv2D(64,(1, 1), activation='relu', padding='same' )(inputs)
 tower_1 = Conv2D(64,(2, 2), activation='relu', padding='same' )(tower_1)
 tower_1 = Conv2D(64,(3, 3), activation='relu', padding='same' )(tower_1)
 tower_1 = MaxPool2D(pool_size=(3,3), strides=(2,2))(tower_1)
 tower_1 = BatchNormalization()(tower_1)
 tower_2 = Conv2D(64,(2, 2), activation='relu', padding='same' )(inputs)
 tower_2 = Conv2D(64,(3, 3), activation='relu', padding='same' )(tower_2)
 tower_2 = Conv2D(64,(5, 5), activation='relu', padding='same' )(tower_2)
 tower_2 = MaxPool2D(pool_size=(3,3), strides=(2,2))(tower_2)
 tower_2 = BatchNormalization()(tower_2)
 tower_3 = Conv2D(64,(3, 3), activation='relu', padding='same' )(inputs)
 tower_3 = Conv2D(64,(3, 3), activation='relu', padding='same' )(tower_3)
 tower_3 = Conv2D(64,(5, 5), activation='relu', padding='same' )(tower_3)
 tower_3 = MaxPool2D(pool_size=(3,3), strides=(2,2))(tower_3)
 tower_3 = BatchNormalization()(tower_3)
 x = Add()([tower_1, tower_2, tower_3])
 x = Activation('relu' )(x)
 x = Conv2D(128,(5, 5), activation='relu', padding='same' )(x)
 x = MaxPool2D(pool_size=(5,5), strides=(3,3))(x)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = Flatten()(x)
 x = Dense(hidden, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 preds = Dense(10, activation='softmax', name='preds' )(x)
 model = Model(inputs=INPUT, outputs=preds)
 optimizer = Adam(lr=learning_rate)
 model.compile(optimizer=optimizer, loss='sparse_categorical_crossentropy', metrics=['acc'])
 return model
model = new_model()<train_model>
voting_clf_SVM_DT = VotingClassifier(
 estimators = [('SVM',SVM_pipe),('Decision Tree',DT_pipe)],
voting = 'hard' 
)
 cross_val_score(voting_clf_SVM_DT,X,y,cv=10,scoring='accuracy' ).mean()
datagen = ImageDataGenerator(featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=5,
zoom_range=0.05,
shear_range=0.02,
width_shift_range=0.05,
height_shift_range=0.05,
horizontal_flip=False,
vertical_flip=False )<train_model>
voting_clf_NB_DT = VotingClassifier(
 estimators = [('Naive Bayes',NB_pipe),('Decision Tree',DT_pipe)],
voting = 'hard' 
)
 cross_val_score(voting_clf_NB_DT,X,y,cv=10,scoring='accuracy' ).mean()
epochs = 200
batch_size = 128
print("Learning Properties: Epoch:%i \t Batch Size:%i" %(epochs, batch_size))
predict_accumulator = np.zeros(model.predict(test ).shape)
 accumulated_history = []
for i in range(1, 6):
X_train, X_val, Y_train, Y_val = train_test_split(X, Y, test_size=0.20, random_state=random_seed*i)
 model = new_model(100, 0.008/i)
 datagen.fit(X_train)
 history = model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size),
epochs=epochs, validation_data=(X_val, Y_val), verbose=3,
steps_per_epoch=X_train.shape[0]//batch_size,
callbacks=[learning_rate_reduction, es],
workers=4)
 loss, acc = model.evaluate(X, Y)
 if acc > 0.75:
predict_accumulator += model.predict(test)*acc
accumulated_history.append(history)
 print("Current Predictions on fold number %i", i)
 print(*np.argmax(predict_accumulator, axis=1), sep='\t' )<save_to_csv>
X_test = test_data.drop(columns=['Survived'] )
print("Completed Training.")
 results = np.argmax(predict_accumulator, axis=1)
 results = pd.Series(results, name="result")
 print("Saving prediction to output...")
 submission = pd.concat([pd.Series(range(0, test.shape[0]), name="ids"), results], axis=1)
 submission.to_csv('prediction.csv', index=False )<import_modules>
voting_clf_NB_DT.fit(X,y )
import numpy as np
import pandas as pd
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.utils import np_utils
from skimage.transform import rescale
from sklearn import preprocessing
from keras.models import load_model
from keras.layers import BatchNormalization
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
from sklearn.model_selection import train_test_split
from tqdm import tqdm<define_variables>
predictions = voting_clf_NB_DT.predict(X_test)
 predictions = pd.to_numeric(predictions, downcast='integer')
 output = pd.DataFrame({'PassengerId': list(X_test.index), 'Survived': predictions} )
img_rows = 40
img_cols = 40<load_from_csv>
output.to_csv('Submission_NB_DT.csv', index=False )