Kaggle competition: Titanic - Machine Learning from Disaster
def fill_missing_values(df):
df.category_name.fillna(value="Other", inplace=True)
 df.brand_name.fillna(value="missing", inplace=True)
 df.item_description.fillna(value="None", inplace=True)
 return df
train_df = fill_missing_values(train_df)
 test_df = fill_missing_values(test_df )<prepare_x_and_y>
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
train_df["target"] = np.log1p(train_df.price)
 train_df, dev_df = train_test_split(train_df, random_state=347, train_size=0.99)
 Y_train = train_df.target.values.reshape(-11, 1)
 Y_dev = dev_df.target.values.reshape(-1, 1)
 n_trains = train_df.shape[0]
n_devs = dev_df.shape[0]
n_tests = test_df.shape[0]
print("Training on", n_trains, "examples")
 print("Validating on", n_devs, "examples")
 print("Testing on", n_tests, "examples" )<concatenate>
df_train=pd.read_csv('.. /input/titanic/train.csv')
 df_test=pd.read_csv('.. /input/titanic/test.csv' )
full_df = pd.concat([train_df, dev_df, test_df] )<categorify>
PassengerId=df_test['PassengerId']
%%time
print("Processing categorical data...")
 le = LabelEncoder()
le.fit(full_df.category_name)
 full_df.category_name = le.transform(full_df.category_name)
 le.fit(full_df.brand_name)
 full_df.brand_name = le.transform(full_df.brand_name)
 del le<categorify>
df_train.drop(['PassengerId','Name','Ticket'],axis=1,inplace=True)
 df_test.drop(['PassengerId','Name','Ticket'],axis=1,inplace=True )
%%time
print("Transforming text data to sequences...")
 raw_text = np.hstack([full_df.item_description.str.lower() , full_df.name.str.lower() ])
 print(" Fitting tokenizer...")
 tok_raw = Tokenizer()
tok_raw.fit_on_texts(raw_text)
 print(" Transforming text to sequences...")
 full_df['seq_item_description'] = tok_raw.texts_to_sequences(full_df.item_description.str.lower())
 full_df['seq_name'] = tok_raw.texts_to_sequences(full_df.name.str.lower())
 del tok_raw<define_variables>
df_train.isnull().sum() /len(df_train)*100
MAX_NAME_SEQ = 10
MAX_ITEM_DESC_SEQ = 75
MAX_TEXT = np.max([
np.max(full_df.seq_name.max()),
np.max(full_df.seq_item_description.max()),
])+ 4
MAX_CATEGORY = np.max(full_df.category_name.max())+ 1
MAX_BRAND = np.max(full_df.brand_name.max())+ 1
MAX_CONDITION = np.max(full_df.item_condition_id.max())+ 1<prepare_x_and_y>
df_train.drop(['Cabin'],axis=1,inplace=True)
 df_test.drop(['Cabin'],axis=1,inplace=True )
%%time
def get_keras_data(df):
X = {
'name': pad_sequences(df.seq_name, maxlen=MAX_NAME_SEQ),
'item_desc': pad_sequences(df.seq_item_description, maxlen=MAX_ITEM_DESC_SEQ),
'brand_name': np.array(df.brand_name),
'category_name': np.array(df.category_name),
'item_condition': np.array(df.item_condition_id),
'num_vars': np.array(df[["shipping"]]),
}
return X
train = full_df[:n_trains]
dev = full_df[n_trains:n_trains+n_devs]
test = full_df[n_trains+n_devs:]
X_train = get_keras_data(train)
 X_dev = get_keras_data(dev)
 X_test = get_keras_data(test )<categorify>
df_train.dropna(subset=['Embarked'],inplace=True)
 df_train['Embarked'].isnull().sum()
def new_rnn_model(lr=0.001, decay=0.0):
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 category_name = Input(shape=[1], name="category_name")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 emb_name = Embedding(MAX_TEXT, 20 )(name)
 emb_item_desc = Embedding(MAX_TEXT, 60 )(item_desc)
 emb_brand_name = Embedding(MAX_BRAND, 10 )(brand_name)
 emb_category_name = Embedding(MAX_CATEGORY, 10 )(category_name)
 rnn_layer1 = GRU(16 )(emb_item_desc)
 rnn_layer2 = GRU(8 )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand_name),
Flatten()(emb_category_name),
item_condition,
rnn_layer1,
rnn_layer2,
num_vars,
])
 main_l = Dense(256 )(main_l)
 main_l = Activation('elu' )(main_l)
 main_l = Dense(128 )(main_l)
 main_l = Activation('elu' )(main_l)
 main_l = Dense(64 )(main_l)
 main_l = Activation('elu' )(main_l)
 output = Dense(1, activation="linear" )(main_l)
 model = Model([name, item_desc, brand_name , category_name, item_condition, num_vars], output)
 optimizer = Adam(lr=lr, decay=decay)
 model.compile(loss="mse", optimizer=optimizer)
 return model
model = new_rnn_model()
model.summary()
del model<train_model>
age_train_series=df_train.groupby(['Pclass','Sex'])['Age'].transform('median' )
%%time
BATCH_SIZE = 1024
epochs = 2
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
steps = int(n_trains / BATCH_SIZE)* epochs
lr_init, lr_fin = 0.007, 0.0005
lr_decay = exp_decay(lr_init, lr_fin, steps)
 rnn_model = new_rnn_model(lr=lr_init, decay=lr_decay)
 print("Fitting RNN model to training examples...")
 rnn_model.fit(
 X_train, Y_train, epochs=epochs, batch_size=BATCH_SIZE,
validation_data=(X_dev, Y_dev), verbose=2,
)<predict_on_test>
age_test_series=df_test.groupby(['Pclass','Sex'])['Age'].transform('median' )
%%time
print("Evaluating the model on validation data...")
 Y_dev_preds_rnn = rnn_model.predict(X_dev, batch_size=BATCH_SIZE)
 print(" RMSLE error:", rmsle(Y_dev, Y_dev_preds_rnn))<predict_on_test>
df_train['Age']=df_train['Age'].fillna(age_train_series )
rnn_preds = rnn_model.predict(X_test, batch_size=BATCH_SIZE, verbose=1)
 rnn_preds = np.expm1(rnn_preds )<concatenate>
df_test['Age']=df_test['Age'].fillna(age_test_series )
full_df = pd.concat([train_df, dev_df, test_df] )<data_type_conversions>
df_test.isnull().sum()
%%time
full_df['shipping'] = full_df['shipping'].astype(str)
 full_df['item_condition_id'] = full_df['item_condition_id'].astype(str )<feature_engineering>
Fare_test_series=df_test.groupby(['Pclass'])['Fare'].transform('median')
 df_test['Fare']=df_test['Fare'].fillna(Fare_test_series )
%%time
print("Vectorizing data...")
 default_preprocessor = CountVectorizer().build_preprocessor()
def build_preprocessor(field):
field_idx = list(full_df.columns ).index(field)
 return lambda x: default_preprocessor(x[field_idx])
 vectorizer = FeatureUnion([ 
('name', CountVectorizer(
 ngram_range=(1, 2),
max_features=50000,
preprocessor=build_preprocessor('name'))), 
('category_name', CountVectorizer(
 token_pattern='.+',
preprocessor=build_preprocessor('category_name'))), 
('brand_name', CountVectorizer(
 token_pattern='.+',
preprocessor=build_preprocessor('brand_name'))), 
('shipping', CountVectorizer(
 token_pattern='\d+',
preprocessor=build_preprocessor('shipping'))), 
('item_condition_id', CountVectorizer(
 token_pattern='\d+',
preprocessor=build_preprocessor('item_condition_id'))), 
('item_description', TfidfVectorizer(
 ngram_range=(1, 3),
max_features=100000,
preprocessor=build_preprocessor('item_description'))),
])
 X = vectorizer.fit_transform(full_df.values)
 X_train = X[:n_trains]
X_dev = X[n_trains:n_trains+n_devs]
X_test = X[n_trains+n_devs:]
print(X.shape, X_train.shape, X_dev.shape, X_test.shape )<train_model>
df_train=pd.get_dummies(df_train,drop_first=True)
%%time
print("Fitting Ridge model on training examples...")
 ridge_model = Ridge(
 solver='auto', fit_intercept=True, alpha=0.5,
max_iter=100, normalize=False, tol=0.05, 
)
 ridge_model.fit(X_train, Y_train )<predict_on_test>
df_test=pd.get_dummies(df_test,drop_first=True )
Y_dev_preds_ridge = ridge_model.predict(X_dev)
 Y_dev_preds_ridge = Y_dev_preds_ridge.reshape(-1, 1)
 print("RMSL error on dev set:", rmsle(Y_dev, Y_dev_preds_ridge))<predict_on_test>
X = df_train.drop('Survived',axis=1)
 y = df_train['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.7, random_state=42)
 X_train.shape, X_test.shape
%%time
ridge_preds = ridge_model.predict(X_test)
 ridge_preds = np.expm1(ridge_preds )<compute_train_metric>
from sklearn.ensemble import RandomForestClassifier
def aggregate_predicts(Y1, Y2):
assert Y1.shape == Y2.shape
ratio = 0.63
return Y1 * ratio + Y2 *(1.0 - ratio)
 Y_dev_preds = aggregate_predicts(Y_dev_preds_rnn, Y_dev_preds_ridge)
 print("RMSL error for RNN + Ridge on dev set:", rmsle(Y_dev, Y_dev_preds))<save_to_csv>
rf=RandomForestClassifier(random_state=42,n_estimators=100,max_depth=4,min_samples_leaf=15,max_features=3)
 rf.fit(X_train,y_train )
preds = aggregate_predicts(rnn_preds, ridge_preds)
 submission = pd.DataFrame({
"test_id": test_df.test_id,
"price": preds.reshape(-1),
})
 submission.to_csv("./rnn_ridge_submission.csv", index=False )<load_from_csv>
accuracy_score(y_test,rf.predict(X_test))
train=pd.read_table('.. /input/train.tsv')
 test=pd.read_table('.. /input/test.tsv' )<feature_engineering>
predictions=rf.predict(df_test)
 titanic_4=pd.DataFrame({'PassengerId':PassengerId,'Survived':predictions})
 titanic_4.to_csv('My_4th_submission',index=False)
<feature_engineering><EOS> 
pd.read_csv('My_4th_submission' )