Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<categorify>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelEncoder, MinMaxScaler
print("Handling missing values...")
 def handle_missing(dataset):
dataset.category_name.fillna(value="missing", inplace=True)
 dataset.brand_name.fillna(value="missing", inplace=True)
 dataset.item_description.fillna(value="missing", inplace=True)
 return(dataset)
 train = handle_missing(train)
 test = handle_missing(test)
 print(train.shape)
 print(test.shape )<categorify>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print('The size of the training set: ', train.shape)
 print('The size of the test set is: ' ,test.shape )
print("Handling categorical variables...")
 le = LabelEncoder()
le.fit(np.hstack([train.category_name, test.category_name]))
train.category_name = le.transform(train.category_name)
 test.category_name = le.transform(test.category_name)
 le.fit(np.hstack([train.brand_name, test.brand_name]))
train.brand_name = le.transform(train.brand_name)
 test.brand_name = le.transform(test.brand_name)
 del le
train.head(3 )<feature_engineering>
Passenger_Id = test['PassengerId']
print("Text to seq process...")
 raw_text = np.hstack([train.item_description.str.lower() , train.name.str.lower() ])
 print(" Fitting tokenizer...")
 tok_raw = Tokenizer()
tok_raw.fit_on_texts(raw_text)
 print(" Transforming text to seq...")
 train["seq_item_description"] = tok_raw.texts_to_sequences(train.item_description.str.lower())
 test["seq_item_description"] = tok_raw.texts_to_sequences(test.item_description.str.lower())
 train["seq_name"] = tok_raw.texts_to_sequences(train.name.str.lower())
 test["seq_name"] = tok_raw.texts_to_sequences(test.name.str.lower())
 train.head(3 )<define_variables>
train_rowsize = train.shape[0]
test_rowsize = test.shape[0]
MAX_NAME_SEQ = 10
MAX_ITEM_DESC_SEQ = 75
MAX_TEXT = np.max([np.max(train.seq_name.max())
 , np.max(test.seq_name.max())
 , np.max(train.seq_item_description.max())
 , np.max(test.seq_item_description.max())])+2
MAX_CATEGORY = np.max([train.category_name.max() , test.category_name.max() ])+1
MAX_BRAND = np.max([train.brand_name.max() , test.brand_name.max() ])+1
MAX_CONDITION = np.max([train.item_condition_id.max() , test.item_condition_id.max() ])+1<normalization>
train.dtypes.value_counts()
train["target"] = np.log(train.price+1)
 target_scaler = MinMaxScaler(feature_range=(-1, 1))
train["target"] = target_scaler.fit_transform(train.target.reshape(-1,1))
pd.DataFrame(train.target ).hist()<split>
test.dtypes.value_counts()
dtrain, dvalid = train_test_split(train, random_state=123, train_size=0.99)
 print(dtrain.shape)
 print(dvalid.shape )<categorify>
data = pd.concat(( train, test))
data.drop('Survived', axis = 1, inplace = True)
 data.drop('PassengerId', axis = 1, inplace = True )
def get_keras_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ)
 ,'item_desc': pad_sequences(dataset.seq_item_description, maxlen=MAX_ITEM_DESC_SEQ)
 ,'brand_name': np.array(dataset.brand_name)
 ,'category_name': np.array(dataset.category_name)
 ,'item_condition': np.array(dataset.item_condition_id)
 ,'num_vars': np.array(dataset[["shipping"]])
 }
return X
X_train = get_keras_data(dtrain)
 X_valid = get_keras_data(dvalid)
 X_test = get_keras_data(test )<compute_train_metric>
data.Sex.value_counts(normalize = True )
def get_callbacks(filepath, patience=2):
es = EarlyStopping('val_loss', patience=patience, mode="min")
 msave = ModelCheckpoint(filepath, save_best_only=True)
 return [es, msave]
def rmsle_cust(y_true, y_pred):
first_log = K.log(K.clip(y_pred, K.epsilon() , None)+ 1.)
second_log = K.log(K.clip(y_true, K.epsilon() , None)+ 1.)
return K.sqrt(K.mean(K.square(first_log - second_log), axis=-1))
def get_model() :
dr_r = 0.1
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 category_name = Input(shape=[1], name="category_name")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 emb_name = Embedding(MAX_TEXT, 50 )(name)
 emb_item_desc = Embedding(MAX_TEXT, 50 )(item_desc)
 emb_brand_name = Embedding(MAX_BRAND, 10 )(brand_name)
 emb_category_name = Embedding(MAX_CATEGORY, 10 )(category_name)
 emb_item_condition = Embedding(MAX_CONDITION, 5 )(item_condition)
 rnn_layer1 = GRU(16 )(emb_item_desc)
 rnn_layer2 = GRU(8 )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand_name)
 , Flatten()(emb_category_name)
 , Flatten()(emb_item_condition)
 , rnn_layer1
, rnn_layer2
, num_vars
])
 main_l = Dropout(dr_r )(Dense(128 )(main_l))
main_l = Dropout(dr_r )(Dense(64 )(main_l))
output = Dense(1, activation="linear" )(main_l)
 model = Model([name, item_desc, brand_name
, category_name, item_condition, num_vars], output)
 model.compile(loss="mse", optimizer="adam", metrics=["mae", rmsle_cust])
 return model
model = get_model()
model.summary()<train_model>
embarked_counts = data.Embarked.value_counts(normalize = True)
 embarked_counts
BATCH_SIZE = 20000
epochs = 2
model = get_model()
model.fit(X_train, dtrain.target, epochs=epochs, batch_size=BATCH_SIZE, validation_data=(X_valid, dvalid.target)
 , verbose=1 )<compute_train_metric>
null_values = data.columns[data.isnull().any() ]
null_features = data[null_values].isnull().sum().sort_values(ascending = False)
 missing_data = pd.DataFrame({'No of Nulls' :null_features})
 missing_data
val_preds = model.predict(X_valid)
 val_preds = target_scaler.inverse_transform(val_preds)
 val_preds = np.exp(val_preds)+1
y_true = np.array(dvalid.price.values)
 y_pred = val_preds[:,0]
v_rmsle = rmsle(y_true, y_pred)
 print(" RMSLE error on dev test: "+str(v_rmsle))<predict_on_test>
test.isnull().sum()
preds = model.predict(X_test, batch_size=BATCH_SIZE)
 preds = target_scaler.inverse_transform(preds)
 preds = np.exp(preds)-1
submission = test[["test_id"]]
submission["price"] = preds<save_to_csv>
test.isnull().sum()
submission.to_csv("sample_submission.csv", index=False)
 submission.price.hist()<import_modules>
train.isnull().sum()
import numpy as np
import pandas as pd<load_from_csv>
train.isnull().sum()
train = pd.read_csv('.. /input/train.tsv', sep='\t')
 test = pd.read_csv('.. /input/test.tsv', sep='\t')
 train["source"] = "train"
test["source"] = "test"
train = train.rename(columns={"train_id": "id"})
 test = test.rename(columns={"test_id": "id"})
 data = pd.concat([train,test] )<string_transform>
data["Embarked"] = data["Embarked"].fillna('S' )
def split_cat(text):
try: return text.split("/")
 except: return("No Label", "No Label", "No Label" )<feature_engineering>
data[data['Fare'].isnull() ]
data['general_cat'], data['subcat_1'], data['subcat_2'] = \
zip(*data['category_name'].apply(lambda x: split_cat(x)) )<drop_column>
def fill_missing_fare(df):
median_fare=df[(df['Pclass'] == 3)&(df['Embarked'] == 'S')]['Fare'].median()
df["Fare"] = df["Fare"].fillna(median_fare)
 return df
data=fill_missing_fare(data )
data.drop('category_name',axis=1,inplace=True )<correct_missing_values>
sizeof_null = data["Age"].isnull().sum()
rand_age = np.random.randint(0, 80, size = sizeof_null )
data.brand_name = data.brand_name.fillna("None" )<categorify>
age_slice = data["Age"].copy()
age_slice[np.isnan(age_slice)] = rand_age
data["Age"] = age_slice
data["Age"] = data["Age"].astype(int )
le = LabelEncoder()
labels = ["brand_name","general_cat","subcat_1","subcat_2"]
for label in labels:
data[label] = data[label].astype(str)
 le.fit(np.hstack(data[label]))
data[label] = le.transform(data[label] )<feature_engineering>
data['Age'] = data['Age'].astype(int)
 data.loc[ data['Age'] <= 18, 'Age'] = 0
data.loc[(data['Age'] > 18)&(data['Age'] <= 35), 'Age'] = 1
data.loc[(data['Age'] > 35)&(data['Age'] <= 60), 'Age'] = 2
data.loc[(data['Age'] > 60)&(data['Age'] <= 80), 'Age'] = 3
data['Age'].value_counts()
data.general_cat = data.general_cat.astype(str)
 vectorizer = CountVectorizer(token_pattern='\d+')
 x = vectorizer.fit_transform(data.general_cat )<data_type_conversions>
data['Deck'] = data['Deck'].fillna('H' )
data.item_description = data.item_description.fillna("None")
 data.item_condition_id = data.item_condition_id.astype(str)
 data.shipping = data.shipping.astype(str)
 data.general_cat = data.general_cat.astype(str)
 data.subcat_1 = data.subcat_1.astype(str)
 data.subcat_2 = data.subcat_2.astype(str)
 data.brand_name = data.brand_name.astype(str )<feature_engineering>
data["FamilySize"] = data["SibSp"] + data["Parch"]+ 1
data['FamilySize'].value_counts()
default_preprocessor = CountVectorizer().build_preprocessor()
def preprocessor(field):
field_idx = list(data.columns ).index(field)
 return lambda x: default_preprocessor(x[field_idx])
 vectorizer = FeatureUnion([ 
('name', CountVectorizer(
 ngram_range=(1, 2),
max_features=50000,
preprocessor = preprocessor('name'))), 
('general_cat', CountVectorizer(
 token_pattern='\d+',
preprocessor=preprocessor('general_cat'))), 
('subcat_1', CountVectorizer(
 token_pattern='\d+',
preprocessor=preprocessor('subcat_1'))), 
('subcat_2', CountVectorizer(
 token_pattern='\d+',
preprocessor=preprocessor('subcat_2'))), 
('brand_name', CountVectorizer(
 token_pattern='\d+',
preprocessor=preprocessor('brand_name'))), 
('shipping', CountVectorizer(
 token_pattern='\d+',
preprocessor=preprocessor('shipping'))), 
('item_condition_id', CountVectorizer(
 token_pattern='\d+',
preprocessor=preprocessor('item_condition_id'))), 
('item_description', TfidfVectorizer(
 ngram_range=(1, 3),
max_features=100000,
preprocessor=preprocessor('item_description'))),
] )<feature_engineering>
data.loc[ data['FamilySize'] == 1, 'FSize'] = 'Single family'
data.loc[(data['FamilySize'] > 1)&(data['FamilySize'] <= 5), 'FSize'] = 'Small Family'
data.loc[(data['FamilySize'] > 5), 'FSize'] = ' Extended Family'
X = vectorizer.fit_transform(data.values )<prepare_x_and_y>
le = LabelEncoder()
data['Sex'] = le.fit_transform(data['Sex'])
 data['Embarked'] = le.fit_transform(data['Embarked'])
 data['Deck'] = le.fit_transform(data['Deck'])
 data['FSize'] = le.fit_transform(data['FSize'] )
trainData = X[:train.shape[0]]
target = np.log1p(train.price)
 testData = X[train.shape[0]:]<split>
data.dtypes.value_counts()
X_train, X_valid, y_train, y_valid = train_test_split(trainData, target, test_size=0.3, random_state=0 )<import_modules>
data = data.drop(['Name', 'Ticket','Cabin',], axis = 1 )
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import Ridge
from sklearn.linear_model import RidgeCV<compute_train_metric>
skewed = skew_cols.index.tolist()
data[skewed] = data[skewed].apply(np.log1p )
ridge_model = Ridge(
 solver='auto', fit_intercept=True, alpha=1.0,
max_iter=100, normalize=False, tol=0.05, random_state = 1, 
)
 ridge_model.fit(X_train, y_train)
 val_preds= ridge_model.predict(X_valid)
 score = mean_squared_error(y_valid, np.array(val_preds))
print("Ridge Score: "+str(np.sqrt(score)) )<compute_train_metric>
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier
from sklearn.ensemble import VotingClassifier
ridge_modelCV = RidgeCV(
 fit_intercept=True, alphas=[5.0],
normalize=False, cv = 2, scoring='neg_mean_squared_error', 
)
 ridge_modelCV.fit(X_train, y_train)
 val_preds= ridge_modelCV.predict(X_valid)
 score = mean_squared_error(y_valid, np.array(val_preds))
print("RidgeCV Score: "+str(np.sqrt(score)) )<save_to_csv>
train_new.dtypes.value_counts()
preds = ridge_modelCV.predict(testData)
 submission= test[["id"]]
submission["price"] = np.expm1(preds)
 submission.rename(columns={"id": "test_id"})
 submission.to_csv("submission.csv",header=["test_id","price"], index=False )<import_modules>
test_new.dtypes.value_counts()
from sklearn.linear_model import Ridge
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import OneHotEncoder
from scipy.sparse import hstack<load_from_csv>
n_folds = 5
kf = KFold(n_folds, shuffle = True, random_state = 42 ).get_n_splits(train_new)
train_df = pd.read_csv('.. /input/train.tsv', delimiter='\t')
 test_df = pd.read_csv('.. /input/test.tsv', delimiter='\t' )<define_variables>
y_train = train.Survived
n_folds = 5
def f1_score(model):
kf = KFold(n_folds, shuffle = True, random_state = 42 ).get_n_splits(train_new)
 rmse = np.sqrt(cross_val_score(model, train_new, y_train, scoring = 'f1', cv = kf))
return(rmse )
train_test_border = len(train_df )<concatenate>
logreg = LogisticRegression()
rf = RandomForestClassifier()
gboost = GradientBoostingClassifier()
xgb = XGBClassifier()
lgbm = LGBMClassifier()
train_test_df = pd.concat([train_df.drop(['train_id', 'price'], axis=1), test_df.drop(['test_id'], axis=1)], axis=0 )<count_missing_values>
train_test_df.isnull().sum()<data_type_conversions>
score = f1_score(logreg)
 print("
Logistic regression score: mean = {:.4f} std =({:.4f})
 ".format(score.mean() , score.std()))
train_test_df.category_name = train_test_df.category_name.fillna('Nan/Nan/Nan')
 train_test_df.brand_name = train_test_df.brand_name.fillna('Nan')
 train_test_df.item_description = train_test_df.item_description.fillna('No description yet' )<count_missing_values>
score = f1_score(rf)
 print("
Random Forest score: mean = {:.4f} std =({:.4f})
 ".format(score.mean() , score.std()))
train_test_df.isnull().sum()<string_transform>
score = f1_score(gboost)
 print("
Gradient Boosting Classifier score: mean = {:.4f} std =({:.4f})
 ".format(score.mean() , score.std()))
split_category = np.array([x for x in train_test_df.category_name.str.split('/', 2 ).values] )<feature_engineering>
score = f1_score(xgb)
 print("
eXtreme Gradient BOOSTing score: mean = {:.4f} std =({:.4f})
 ".format(score.mean() , score.std()))
train_test_df['split_category_0'] = split_category[:,0]
train_test_df['split_category_1'] = split_category[:,1]
train_test_df['split_category_2'] = split_category[:,2]
train_test_df['brand_category_2'] = train_test_df.split_category_2.values + '/' + train_test_df.brand_name.values<drop_column>
score = f1_score(lgbm)
 print("
Light Gradient Boosting score: mean = {:.4f} std =({:.4f})
 ".format(score.mean() , score.std()))
train_test_df = train_test_df.drop(['category_name'], axis=1 )<count_values>
all_classifier = VotingClassifier(estimators=[('logreg', logreg),('rf', rf), 
('gboost', gboost),('xgb', xgb), 
('lgbm', lgbm)], voting='soft')
 VC = all_classifier.fit(train_new, y_train )
train_test_df.split_category_0.value_counts()<filter>
score = f1_score(VC)
 print("
Voting Classifier score: mean = {:.4f} std =({:.4f})
 ".format(score.mean() , score.std()))
category_df = train_test_df[train_test_df.split_category_0 == 'Women']<prepare_output>
prediction = VC.predict(test_new )
<feature_engineering><EOS> 
titanic_submission = pd.DataFrame({"PassengerId": test["PassengerId"],
"Survived": prediction})
 titanic_submission.to_csv('Titanic_Submission.csv', index = False)
 titanic_submission.sample(10)