Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<import_modules>
warnings.filterwarnings('ignore' )
from keras.models import Sequential
from keras.layers import Dense, Activation<load_from_csv>
gender_submission = pd.read_csv(".. /input/titanic/gender_submission.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 train = pd.read_csv(".. /input/titanic/train.csv")
 display(test.head(15))
display(train.describe() )
print("reading train files")
 train = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv',encoding='utf-8')
 train = train.replace(r'
',' ', regex=True)
 train = train.replace(r'\',' ', regex=True)
 print(train.head())
 print("Now reading test files")
 test=pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv',encoding='utf-8')
 ’ ).
 ]',re.IGNORECASE)
 special_character_removal = re.compile(r'[^A-Za-z\.\-\?\!\,\
def clean_text(x):
x_ascii = unidecode(x)
 x_clean = special_character_removal.sub('',x_ascii)
 return x_clean
train['clean_text'] = train['comment_text'].apply(lambda x: clean_text(str(x)))
 test['clean_text'] = test['comment_text'].apply(lambda x: clean_text(str(x)))
 X_train = train['clean_text'].fillna("something" ).values
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
X_test = test['clean_text'].fillna("something" ).values 
<feature_engineering>
all_data['Docker_num'] = [cab[:1] if pd.notnull(cab)else "Unknown" for cab in all_data['Cabin']]
all_data['Has_cabin_informed'] = [1 if pd.notnull(cab)else 0 for cab in all_data['Cabin']]
all_data['Title'] = [re.search('\, (.*)\.', name ).group(1)for name in all_data['Name']]
all_data.set_value(all_data['PassengerId']==514, 'Title', ' Mrs')
 dictionary_of_titles = {
" Capt": "Crew",
" Col": "Crew",
" Major": "Crew",
" Dr": "Crew",
" Rev": "Crew",
" Jonkheer": "VIP",
" Don": "VIP",
" Dona": "VIP",
" Sir" : "VIP",
" the Countess":"VIP",
" Lady" : "VIP",
" Mme": "Mrs",
" Ms": "Miss",
" Mrs" : "Mrs",
" Mlle": "Miss",
" Miss" : "Miss",
" Mr" : "Mr",
" Master" : "Master"
}
all_data['Title'] = all_data.Title.map(dictionary_of_titles)
 def fill_missing_age_title(all_data):
class_age_female_miss = all_data[(all_data['Title']=='Miss')]['Age'].dropna().mean()
class_age_female_mrs = all_data[(all_data['Title']=='Mrs')]['Age'].dropna().mean()
class_age_male_master = all_data[(all_data['Title']=='Master')]['Age'].dropna().mean()
class_age_male_mr = all_data[(all_data['Title']=='Mr')]['Age'].dropna().mean()
all_data[all_data['Age'].isnull() ==True].head(5)
 all_data.set_value(all_data[(all_data['Age'].isnull() ==True)&(all_data['Title']=='Miss')].index, 'Age', class_age_female_miss)
 all_data.set_value(all_data[(all_data['Age'].isnull() ==True)&(all_data['Title']=='Mrs')].index, 'Age', class_age_female_mrs)
 all_data.set_value(all_data[(all_data['Age'].isnull() ==True)&(all_data['Title']=='Master')].index, 'Age', class_age_male_master)
 all_data.set_value(all_data[(all_data['Age'].isnull() ==True)&(all_data['Title']=='Mr')].index, 'Age', class_age_male_mr)
 return all_data
all_data = fill_missing_age_title(all_data)
 def age_cat(age):
if age <= 15:
return 1
if age <= 35:
return 2
if age <= 55:
return 3
if age > 55:
return 4
else:
return 0
all_data['Age_cat'] = all_data['Age'].apply(age_cat)
 def fare_cat(fare):
if fare < 15:
return 1
if fare < 35:
return 2
if fare < 100:
return 3
if fare > 100:
return 4
else:
return 0
all_data['Fare_cat'] = all_data['Age'].apply(fare_cat)
 all_data['Relatives'] = all_data['SibSp'] + all_data['Parch']
def fill_families(all_data):
all_data['Last_Name'] = all_data['Name'].apply(lambda x: str.split(x, ",")[0])
 default_survival_chance = 0.5
all_data['Family_Survival'] = default_survival_chance
for grp, grp_df in all_data[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
all_data.loc[all_data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin == 0.0):
all_data.loc[all_data['PassengerId'] == passID, 'Family_Survival'] = 0
all_data['IsAlone'] = 0
all_data.loc[all_data['Relatives'] == 0, 'IsAlone'] = 1
return all_data
all_data = fill_families(all_data)
 all_data.head(5 )
def add_features(df):
df['comment_text'] = df['comment_text'].apply(lambda x:str(x))
df['total_length'] = df['comment_text'].apply(len)
 df['capitals'] = df['comment_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']),
axis=1)
 df['num_words'] = df.comment_text.str.count('\S+')
 df['num_unique_words'] = df['comment_text'].apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df
train = add_features(train)
 test = add_features(test)
 features = train[['caps_vs_length', 'words_vs_unique']].fillna(0)
 test_features = test[['caps_vs_length', 'words_vs_unique']].fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features )<sort_values>
sns.distplot(all_data['Fare'].dropna())
 plt.ylabel('Frequency')
 plt.title('Fare distribution')
 all_data['Fare']=all_data['Fare'].apply(lambda x: np.log(x))
train_comb = train.groupby(COLUMNS)\ 
.size() \ 
.sort_values(ascending=False)\ 
.reset_index() \ 
.rename(columns={0: 'count'})
 train_comb.head(n=10 )<sort_values>
sns.distplot(all_data['Age'].dropna())
 plt.ylabel('Frequency')
 plt.title('Age distribution')
 all_data['Age']=all_data['Age'].apply(lambda x: np.log(x))
print("Feature engineering: Completed" )
train[COLUMNS].corr().abs().unstack().sort_values(ascending=False )<feature_engineering>
X = all_data[:len(train)]
X_test_full = all_data[len(train):]
y = X.Survived
X.drop('Survived', axis=1, inplace=True)
 print(len(all_data), len(X), len(X_test_full))
X_train_full, X_valid_full, y_train, y_valid = train_test_split(X, y, train_size=0.95, test_size=0.05, random_state=0)
 low_cardinality_cols = [cname for cname in X_train_full.columns if X_train_full[cname].nunique() < 10 and
X_train_full[cname].dtype not in ['int64', 'float64']]
print("Low cardinality columns: ", low_cardinality_cols)
 numeric_cols = [cname for cname in X_train_full.columns if X_train_full[cname].dtype in ['int64', 'float64']]
print("Numeric columns: ", numeric_cols)
 my_cols = low_cardinality_cols + numeric_cols
X_train = X_train_full[my_cols].copy()
X_valid = X_valid_full[my_cols].copy()
X_test = X_test_full[my_cols].copy()
X_train = pd.get_dummies(X_train)
 X_valid = pd.get_dummies(X_valid)
 X_test = pd.get_dummies(X_test)
 X_train, X_valid = X_train.align(X_valid, join='left', axis=1)
 X_train, X_test = X_train.align(X_test, join='left', axis=1 )
word_counter = {}
def clean_text(text):
text = re.sub('[{}]'.format(string.punctuation), ' ', text.lower())
 return ' '.join([word for word in text.split() if word not in(eng_stopwords)])
 for categ in CATEGORIES:
d = Counter()
train[train[categ] == 1]['comment_text'].apply(lambda t: d.update(clean_text(t ).split()))
word_counter[categ] = pd.DataFrame.from_dict(d, orient='index')\ 
.rename(columns={0: 'count'})\ 
.sort_values('count', ascending=False)
 for w in word_counter:
wc = word_counter[w]
wordcloud = WordCloud(
 background_color='black',
max_words=200,
max_font_size=100,
random_state=4561
).generate_from_frequencies(wc.to_dict() ['count'])
 fig = plt.figure(figsize=(12, 8))
plt.title(w)
 plt.imshow(wordcloud)
 plt.axis('off')
 plt.show()<string_transform>
def xgb_optimize(X_train, y_train):
xgb1 = xgb()
parameters = {'nthread':[1],
'learning_rate': [.005,.004,.003,.002,.0009, 0.008],
'max_depth': [4, 5, 6, 7],
'min_child_weight': [4, 5, 6],
'silent': [1],
'subsample': [0.5],
'colsample_bytree': [0.7],
'n_estimators': [1000, 2500, 5000, 7500]}
xgb_grid = GridSearchCV(xgb1,
parameters,
cv = 3,
n_jobs = 5,
verbose=True)
 xgb_grid.fit(X_train, y_train.astype(int))
print(xgb_grid.best_score_)
 print(xgb_grid.best_params_)
max_features=20000
maxlen = 50
tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X_train)+ list(X_test))
X_train_sequence = tokenizer.texts_to_sequences(X_train)
 X_test_sequence = tokenizer.texts_to_sequences(X_test)
 x_train = sequence.pad_sequences(X_train_sequence, maxlen=maxlen)
 x_test = sequence.pad_sequences(X_test_sequence, maxlen=maxlen)
<define_variables>
model = xgb(colsample_bytree=0.7, learning_rate=0.0009, max_depth=6, min_child_weight=5, n_estimators=2500,
nthread=1, silent=1, subsample=0.7, random_state=0,
early_stopping_rounds = 10, eval_set=[(X_valid, y_valid)], verbose=False)
 print("Let's the training begin.Plase wait.")
 my_pipeline = Pipeline(steps=[('model', model)])
 my_pipeline.fit(X_train, y_train.astype(int))
print("Training finished! Now let's predict test values.")
 preds_test = my_pipeline.predict(X_test)
 output = pd.DataFrame({'PassengerId': X_test.index+892,
'Survived': preds_test.astype(int)})
 output.to_csv('submission.csv', index=False)
 print("Everything finished correctly!" )
max_features=20000
maxlen = 50<feature_engineering>
scores = cross_val_score(my_pipeline, X_train, y_train,
cv=5,
scoring='accuracy')
 print(scores )
<count_values><EOS> 
output = pd.DataFrame({'PassengerId': X_test.index+892,
'Survived': preds_test.astype(int)})
 output.to_csv('submission.csv', index=False )