Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<create_dataframe>
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import math
write_csv=pd.DataFrame({'test_id':test_df.test_id,
'price':predict_2_price.reshape(predict_2_price.shape[0])} )<create_dataframe>
gender_df= pd.read_csv('.. /input/titanic/gender_submission.csv')
 train_df = pd.read_csv('.. /input/titanic/train.csv')
 print(train_df.isnull().sum(axis = 0))
write_csv2=pd.DataFrame({'test_id':write_csv.test_id,
'price':write_csv.price} )<load_from_csv>
si = SimpleImputer(strategy="most_frequent")
 inputerResult = si.fit_transform(train_df[['Embarked']])
 train_df['Embarked'] = inputerResult
si = SimpleImputer(strategy="mean")
 inputerResult = si.fit_transform(train_df[['Age']])
 train_df['Age'] = inputerResult
train_df["Age"] = train_df['Age'].map(lambda x: '{:.2f}'.format(float(x)))
sam=pd.read_csv('.. /input/sample_submission.csv' )<save_to_csv>
print(train_df.isnull().sum(axis = 0))
write_csv2.to_csv('sample_submission.csv',index=False )<compute_test_metric>
def binAge(row):
if row['Sex'] == 'male':
if float(row['Age'])< 16:
return 'alive'
else:
return 'dead'
else:
return 'itsFemale'
train_df['bin_male_age'] = train_df.apply(binAge, axis=1 )
def rmsle(y_true, yz_pred):
assert len(y_true)== len(y_pred)
 return(np.square(np.log(y_pred + 1)- np.log(y_true + 1)).mean())**0.5 
<load_from_csv>
def withNanny(row):
if float(row['Parch'])== 0 and float(row['Age'])< 18:
return 'with'
elif float(row['Parch'])!= 0 and float(row['Age'])< 18:
return 'without'
else:
return 'adult'
train_df['withNanny'] = train_df.apply(withNanny, axis=1 )
print("Loading data...")
 train = pd.read_csv('.. /input/train.tsv', sep='\t')
 test = pd.read_csv('.. /input/test.tsv', sep='\t')
 train['target'] = np.log1p(train['price'])
 print(train.shape)
 print(test.shape )<categorify>
print("Only {}% of data have Cabin number".format(100 - train_df.Cabin.isnull().sum(axis = 0)*100/len(train_df.Cabin)) )
print("Handling missing values...")
 def handle_missing(dataset):
dataset.category_name.fillna(value="missing", inplace=True)
 dataset.brand_name.fillna(value="missing", inplace=True)
 dataset.item_description.fillna(value="missing", inplace=True)
 return(dataset)
 train = handle_missing(train)
 test = handle_missing(test)
 print(train.shape)
 print(test.shape )<categorify>
features = [ 'Age', 'SibSp', 'Parch', 'Fare']
cat_features = ['Embarked', 'Sex','Pclass' , 'withNanny']
todo = ['Cabin', 'Ticket']
print("Handling categorical variables...")
 le = LabelEncoder()
le.fit(np.hstack([train.category_name, test.category_name]))
train['category'] = le.transform(train.category_name)
 test['category'] = le.transform(test.category_name)
 le.fit(np.hstack([train.brand_name, test.brand_name]))
train['brand_name'] = le.transform(train.brand_name)
 test['brand_name'] = le.transform(test.brand_name)
 del le
train.head(3)
<feature_engineering>
base_model = train_df[features]
cat_df = train_df[cat_features]
print("Text to seq process...")
 print(" Fitting tokenizer...")
 raw_text = np.hstack([train.category_name.str.lower() ,
train.item_description.str.lower() ,
train.name.str.lower() ])
 tok_raw = Tokenizer()
tok_raw.fit_on_texts(raw_text)
 print(" Transforming text to seq...")
 train["seq_category_name"] = tok_raw.texts_to_sequences(train.category_name.str.lower())
 test["seq_category_name"] = tok_raw.texts_to_sequences(test.category_name.str.lower())
 train["seq_item_description"] = tok_raw.texts_to_sequences(train.item_description.str.lower())
 test["seq_item_description"] = tok_raw.texts_to_sequences(test.item_description.str.lower())
 train["seq_name"] = tok_raw.texts_to_sequences(train.name.str.lower())
 test["seq_name"] = tok_raw.texts_to_sequences(test.name.str.lower())
 train.head(3 )<define_variables>
data = pd.get_dummies(cat_df, columns=['Embarked'], prefix="Embarked", drop_first=True)
 data = pd.get_dummies(data, columns=['Sex'], prefix="Sex", drop_first=True)
 data = pd.get_dummies(data, columns=['Pclass'], prefix="Pclass", drop_first=True)
 data = pd.get_dummies(data, columns=['withNanny'], prefix="withNanny", drop_first=True )
MAX_NAME_SEQ = 15
MAX_ITEM_DESC_SEQ = 65
MAX_CATEGORY_NAME_SEQ = 20
MAX_TEXT = np.max([np.max(train.seq_name.max())
 , np.max(test.seq_name.max())
 , np.max(train.seq_category_name.max())
 , np.max(test.seq_category_name.max())
 , np.max(train.seq_item_description.max())
 , np.max(test.seq_item_description.max())])+2
MAX_CATEGORY = np.max([train.category.max() , test.category.max() ])+1
MAX_BRAND = np.max([train.brand_name.max() , test.brand_name.max() ])+1
MAX_CONDITION = np.max([train.item_condition_id.max() , test.item_condition_id.max() ])+1<normalization>
X_train, X_test, y_train, y_test = train_test_split(data, train_df.Survived, test_size = 0.25, random_state = 0 )
train["target"] = np.log(train.price+1)
 target_scaler = MinMaxScaler(feature_range=(-1, 1))
train["target"] = target_scaler.fit_transform(train.target.reshape(-1,1))
pd.DataFrame(train.target ).hist()<split>
classifier = SVC(kernel='rbf', random_state = 123)
 classifier.fit(X_train, y_train )
dtrain, dvalid = train_test_split(train, random_state=123, train_size=0.985)
 print(dtrain.shape)
 print(dvalid.shape )<string_transform>
y_pred = classifier.predict(X_test )
def get_keras_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ)
 ,'item_desc': pad_sequences(dataset.seq_item_description, maxlen=MAX_ITEM_DESC_SEQ)
 ,'brand_name': np.array(dataset.brand_name)
 ,'category': np.array(dataset.category)
 ,'category_name': pad_sequences(dataset.seq_category_name
, maxlen=MAX_CATEGORY_NAME_SEQ)
 ,'item_condition': np.array(dataset.item_condition_id)
 ,'num_vars': np.array(dataset[["shipping"]])
 }
return X
X_train = get_keras_data(dtrain)
 X_valid = get_keras_data(dvalid)
 X_test = get_keras_data(test )<compute_train_metric>
accuracy_score(y_test, y_pred )
def get_callbacks(filepath, patience=2):
es = EarlyStopping('val_loss', patience=patience, mode="min")
 msave = ModelCheckpoint(filepath, save_best_only=True)
 return [es, msave]
def rmsle_cust(y_true, y_pred):
first_log = K.log(K.clip(y_pred, K.epsilon() , None)+ 1.)
second_log = K.log(K.clip(y_true, K.epsilon() , None)+ 1.)
return K.sqrt(K.mean(K.square(first_log - second_log), axis=-1))
def get_model() :
dr_r = 0.05
dr_d = 0.05
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 category = Input(shape=[1], name="category")
 category_name = Input(shape=[X_train["category_name"].shape[1]],
name="category_name")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 emb_size = 55
emb_name = Embedding(MAX_TEXT, 60 )(name)
 emb_item_desc = Embedding(MAX_TEXT, 60 )(item_desc)
 emb_category_name = Embedding(MAX_TEXT, emb_size//3 )(category_name)
 emb_brand_name = Embedding(MAX_BRAND, 10 )(brand_name)
 emb_category = Embedding(MAX_CATEGORY, 10 )(category)
 emb_item_condition = Embedding(MAX_CONDITION, 5 )(item_condition)
 rnn_layer1 = GRU(16 )(emb_item_desc)
 rnn_layer2 = GRU(16 )(emb_category_name)
 rnn_layer3 = GRU(8 )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand_name)
 , Flatten()(emb_category)
 , Flatten()(emb_item_condition)
 , rnn_layer1
, rnn_layer2
, rnn_layer3
, num_vars
])
 main_l = Dropout(dr_r )(Dense(512,activation='elu' )(main_l))
main_l = Dropout(dr_d )(Dense(16,activation='elu' )(main_l))
output = Dense(1, activation="linear" )(main_l)
 model = Model([name, item_desc, brand_name
,category, category_name, item_condition, num_vars], output)
 model.compile(loss="mse", optimizer="adam", metrics=["mae", rmsle_cust])
 return model
model = get_model()
model.summary()<train_model>
test_df = pd.read_csv('.. /input/titanic/test.csv')
 print(test_df.isnull().sum(axis = 0))
BATCH_SIZE = 512 * 3
epochs = 2
model = get_model()
model.fit(X_train, dtrain.target, epochs=epochs, batch_size=BATCH_SIZE
, validation_data=(X_valid, dvalid.target)
 , verbose=1 )<compute_train_metric>
si = SimpleImputer(strategy="mean")
 inputerResult = si.fit_transform(test_df[['Age']])
 test_df['Age'] = inputerResult
val_preds = model.predict(X_valid)
 val_preds = target_scaler.inverse_transform(val_preds)
 val_preds = np.exp(val_preds)+1
y_true = np.array(dvalid.price.values)
 y_pred = val_preds[:,0]
v_rmsle = rmsle(y_true, y_pred)
 print(" RMSLE error on dev test: "+str(v_rmsle))<predict_on_test>
test_df['bin_male_age'] = test_df.apply(binAge, axis=1)
 test_df['withNanny'] = test_df.apply(withNanny, axis=1 )
preds = model.predict(X_test, batch_size=BATCH_SIZE)
 preds = target_scaler.inverse_transform(preds)
 preds = np.exp(preds)-1
submission = test[["test_id"]]
submission["price"] = preds<save_to_csv>
base_model = test_df[features]
cat_df = test_df[cat_features]
data = pd.get_dummies(cat_df, columns=['Embarked'], prefix="Embarked", drop_first=True)
 data = pd.get_dummies(data, columns=['Sex'], prefix="Sex", drop_first=True)
 data = pd.get_dummies(data, columns=['Pclass'], prefix="Pclass", drop_first=True)
 data = pd.get_dummies(data, columns=['withNanny'], prefix="withNanny", drop_first=True )
submission.to_csv("./NNsubmission.csv", index=False)
 submission.price.hist() 
<init_hyperparams>
y_pred = classifier.predict(data )
start_time = time.time()
def avg_predictions(L):
N = len(L)
 f = L[0]
for p in range(1,N):
f = f + L[p]
f =(1/N)*f
return f
def define_model(data, nodes1, nodes2, drop1, drop2):
x = Input(shape =(data.shape[1],), dtype = 'float32', sparse = True)
 d1 = Dense(nodes1, activation='relu' )(x)
 d2 = Dropout(drop1 )(d1)
 d3 = Dense(nodes2, activation='sigmoid' )(d2)
 d4 = Dropout(drop2 )(d3)
 out= Dense(1, activation = 'linear' )(d4)
 model = Model(x,out)
 return model
def preprocess(num_brands, name_min, max_feat_desc, ngrams):
print("Preprocessing Data...")
 NUM_BRANDS = num_brands
NAME_MIN_DF = name_min
MAX_FEAT_DESCP = max_feat_desc
df_train = pd.read_csv('.. /input/train.tsv', sep='\t')
 df_train = df_train.reindex(np.random.permutation(df_train.index))
df_train.reset_index(inplace=True, drop=True)
 df_test = pd.read_csv('.. /input/test.tsv', sep='\t')
 df = pd.concat([df_train, df_test], 0)
 nrow_train = df_train.shape[0]
Y = np.log1p(df_train["price"])
 del df_train
gc.collect()
df["category_name"] = df["category_name"].fillna("Other" ).astype("category")
 df["brand_name"] = df["brand_name"].fillna("unknown")
 pop_brands = df["brand_name"].value_counts().index[:NUM_BRANDS]
df.loc[~df["brand_name"].isin(pop_brands), "brand_name"] = "Other"
df["item_description"] = df["item_description"].fillna("None")
 df["item_condition_id"] = df["item_condition_id"].astype("category")
 df["brand_name"] = df["brand_name"].astype("category")
 count = CountVectorizer(min_df=NAME_MIN_DF)
 X_name = count.fit_transform(df["name"])
 unique_categories = pd.Series("/".join(df["category_name"].unique().astype("str")).split("/")).unique()
count_category = CountVectorizer()
X_category = count_category.fit_transform(df["category_name"])
 count_descp = TfidfVectorizer(max_features = MAX_FEAT_DESCP,
ngram_range =(1, ngrams),
stop_words = "english")
 X_descp = count_descp.fit_transform(df["item_description"])
 vect_brand = LabelBinarizer(sparse_output=True)
 X_brand = vect_brand.fit_transform(df["brand_name"])
 X_dummies = scipy.sparse.csr_matrix(pd.get_dummies(df[[
"item_condition_id", "shipping"]], sparse = True ).values)
 X = scipy.sparse.hstack(( X_dummies,
X_descp,
X_brand,
X_category,
X_name)).tocsr()
return X[:nrow_train], Y, X[nrow_train:], df_test
def set_split(X_data, y_data, test_size):
N = int(X_data.shape[0]*(1-test_size))
return(X_data[:N], X_data[N:], y_data[:N], y_data[N:])
 def hms_string(sec_elapsed):
h = int(sec_elapsed /(60 * 60))
m = int(( sec_elapsed %(60 * 60)) / 60)
 s = sec_elapsed % 60
return "{}:{:>02}:{:>05.2f}".format(h, m, s)
 start_time = time.time()
X, Y, X_test, df_test = preprocess(2500, 10, 50000, 3)
 x_train, x_val, y_train, y_val = set_split(X, Y, test_size=0.10)
 elapsed_time = time.time() - start_time
print("Preprocessing Time: {}".format(hms_string(elapsed_time)))
 tpoint1 = time.time()
print("Fitting Model...")
 nodes1 = 64
nodes2 = 32
drop1 = 0.30
drop2 = 0.25
print("Training Model...")
 model = define_model(x_train, nodes1, nodes2, drop1, drop2)
 monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, patience=3, verbose=1, mode='auto')
 model.compile(loss='mean_squared_error', optimizer='adam')
 model.fit(x=x_train, y=y_train,
batch_size=600,
callbacks=[monitor],
validation_data=(x_val, y_val),
epochs=10, verbose=0)
 tpoint2 = time.time()
print("Time Training: {}".format(hms_string(tpoint2-tpoint1)))
 pred = model.predict(x=X_test, batch_size=8000, verbose=0)
 tpoint3 = time.time()
print("Time for Predicting: {}".format(hms_string(tpoint3-tpoint2)))
 df_test["price"] = np.expm1(pred)
 df_test[["test_id", "price"]].to_csv("submission_NN.csv", index = False)
 elapsed_time = time.time() - start_time
print("Total Time: {}".format(hms_string(elapsed_time)))
<import_modules>
resultDF = pd.DataFrame(index=test_df.index )
import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder
import xgboost as xgb
import operator<load_from_csv>
resultDF['PassengerId'] = pd.Series(test_df['PassengerId'], index=resultDF.index, dtype=int)
 resultDF['Survived'] = pd.Series(y_pred, index=resultDF.index, dtype=int )
df_train = pd.read_csv('.. /input/train_users_2.csv')
 df_test = pd.read_csv('.. /input/test_users.csv')
 labels = df_train['country_destination'].values
df_train = df_train.drop(['country_destination'], axis=1)
 id_test = df_test['id']
piv_train = df_train.shape[0]<concatenate>
for index, row in test_df.iterrows() :
resultDF['PassengerId'][index] = row.PassengerId
<feature_engineering><EOS> 
resultDF.to_csv('out.csv', index=False )