Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
%matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
 GradientBoostingClassifier, ExtraTreesClassifier)
PROCESS_TWEETS = False
if PROCESS_TWEETS:
total['text'] = total['text'].apply(lambda x: x.lower())
 total['text'] = total['text'].apply(lambda x: re.sub(r'https?://\S+|www\.\S+', '', x, flags = re.MULTILINE))
total['text'] = total['text'].apply(remove_punctuation)
 total['text'] = total['text'].apply(remove_stopwords)
 total['text'] = total['text'].apply(remove_less_than)
 total['text'] = total['text'].apply(remove_non_alphabet)
 total['text'] = total['text'].apply(spell_check )<define_variables>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 PassengerId = test['PassengerId']
train.head(3 )
contractions = {
"ain't": "am not / are not / is not / has not / have not",
"aren't": "are not / am not",
"can't": "cannot",
"can't've": "cannot have",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"couldn't've": "could not have",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hadn't've": "had not have",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he had / he would",
"he'd've": "he would have",
"he'll": "he shall / he will",
"he'll've": "he shall have / he will have",
"he's": "he has / he is",
"how'd": "how did",
"how'd'y": "how do you",
"how'll": "how will",
"how's": "how has / how is / how does",
"I'd": "I had / I would",
"I'd've": "I would have",
"I'll": "I shall / I will",
"I'll've": "I shall have / I will have",
"I'm": "I am",
"I've": "I have",
"isn't": "is not",
"it'd": "it had / it would",
"it'd've": "it would have",
"it'll": "it shall / it will",
"it'll've": "it shall have / it will have",
"it's": "it has / it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"mightn't've": "might not have",
"must've": "must have",
"mustn't": "must not",
"mustn't've": "must not have",
"needn't": "need not",
"needn't've": "need not have",
"o'clock": "of the clock",
"oughtn't": "ought not",
"oughtn't've": "ought not have",
"shan't": "shall not",
"sha'n't": "shall not",
"shan't've": "shall not have",
"she'd": "she had / she would",
"she'd've": "she would have",
"she'll": "she shall / she will",
"she'll've": "she shall have / she will have",
"she's": "she has / she is",
"should've": "should have",
"shouldn't": "should not",
"shouldn't've": "should not have",
"so've": "so have",
"so's": "so as / so is",
"that'd": "that would / that had",
"that'd've": "that would have",
"that's": "that has / that is",
"there'd": "there had / there would",
"there'd've": "there would have",
"there's": "there has / there is",
"they'd": "they had / they would",
"they'd've": "they would have",
"they'll": "they shall / they will",
"they'll've": "they shall have / they will have",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd": "we had / we would",
"we'd've": "we would have",
"we'll": "we will",
"we'll've": "we will have",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what shall / what will",
"what'll've": "what shall have / what will have",
"what're": "what are",
"what's": "what has / what is",
"what've": "what have",
"when's": "when has / when is",
"when've": "when have",
"where'd": "where did",
"where's": "where has / where is",
"where've": "where have",
"who'll": "who shall / who will",
"who'll've": "who shall have / who will have",
"who's": "who has / who is",
"who've": "who have",
"why's": "why has / why is",
"why've": "why have",
"will've": "will have",
"won't": "will not",
"won't've": "will not have",
"would've": "would have",
"wouldn't": "would not",
"wouldn't've": "would not have",
"y'all": "you all",
"y'all'd": "you all would",
"y'all'd've": "you all would have",
"y'all're": "you all are",
"y'all've": "you all have",
"you'd": "you had / you would",
"you'd've": "you would have",
"you'll": "you shall / you will",
"you'll've": "you shall have / you will have",
"you're": "you are",
"you've": "you have"
}
contractions_re = re.compile('(%s)' % '|'.join(contractions.keys()))
def expand_contractions(s, contractions = contractions):
def replace(match):
return contractions[match.group(0)]
return contractions_re.sub(replace, s)
 expand_contractions("can't stop won't stop" )<feature_engineering>
full_data = [train, test]
train['Name_length'] = train['Name'].apply(len)
 test['Name_length'] = test['Name'].apply(len)
 train['Has_Cabin'] = train["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 test['Has_Cabin'] = test["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 for dataset in full_data:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
for dataset in full_data:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
for dataset in full_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S')
 for dataset in full_data:
dataset['Fare'] = dataset['Fare'].fillna(train['Fare'].median())
 train['CategoricalFare'] = pd.qcut(train['Fare'], 4)
 for dataset in full_data:
age_avg = dataset['Age'].mean()
age_std = dataset['Age'].std()
age_null_count = dataset['Age'].isnull().sum()
age_null_random_list = np.random.randint(age_avg - age_std, age_avg + age_std, size=age_null_count)
 dataset['Age'][np.isnan(dataset['Age'])] = age_null_random_list
dataset['Age'] = dataset['Age'].astype(int)
 train['CategoricalAge'] = pd.cut(train['Age'], 5)
 def get_title(name):
title_search = re.search('([A-Za-z]+)\.', name)
 if title_search:
return title_search.group(1)
 return ""
for dataset in full_data:
dataset['Title'] = dataset['Name'].apply(get_title)
 for dataset in full_data:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 for dataset in full_data:
dataset['Sex'] = dataset['Sex'].map({'female': 0, 'male': 1} ).astype(int)
 title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] 						 = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] 							 = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 dataset.loc[ dataset['Age'] <= 16, 'Age'] 					 = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] = 4 ;
total['text'] = total['text'].apply(expand_contractions )<categorify>
drop_elements = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'SibSp']
train = train.drop(drop_elements, axis = 1)
 train = train.drop(['CategoricalAge', 'CategoricalFare'], axis = 1)
 test = test.drop(drop_elements, axis = 1 )
def clean(tweet):
tweet = re.sub(r"tnwx", "Tennessee Weather", tweet)
 tweet = re.sub(r"azwx", "Arizona Weather", tweet)
 tweet = re.sub(r"alwx", "Alabama Weather", tweet)
 tweet = re.sub(r"wordpressdotcom", "wordpress", tweet)
 tweet = re.sub(r"gawx", "Georgia Weather", tweet)
 tweet = re.sub(r"scwx", "South Carolina Weather", tweet)
 tweet = re.sub(r"cawx", "California Weather", tweet)
 tweet = re.sub(r"usNWSgov", "United States National Weather Service", tweet)
 tweet = re.sub(r"MH370", "Malaysia Airlines Flight 370", tweet)
 tweet = re.sub(r"okwx", "Oklahoma City Weather", tweet)
 tweet = re.sub(r"arwx", "Arkansas Weather", tweet)
 tweet = re.sub(r"lmao", "laughing my ass off", tweet)
 tweet = re.sub(r"amirite", "am I right", tweet)
 tweet = re.sub(r"w/e", "whatever", tweet)
 tweet = re.sub(r"w/", "with", tweet)
 tweet = re.sub(r"USAgov", "USA government", tweet)
 tweet = re.sub(r"recentlu", "recently", tweet)
 tweet = re.sub(r"Ph0tos", "Photos", tweet)
 tweet = re.sub(r"exp0sed", "exposed", tweet)
 tweet = re.sub(r"<3", "love", tweet)
 tweet = re.sub(r"amageddon", "armageddon", tweet)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"WindStorm", "Wind Storm", tweet)
 tweet = re.sub(r"16yr", "16 year", tweet)
 tweet = re.sub(r"TRAUMATISED", "traumatized", tweet)
 tweet = re.sub(r"IranDeal", "Iran Deal", tweet)
 tweet = re.sub(r"ArianaGrande", "Ariana Grande", tweet)
 tweet = re.sub(r"camilacabello97", "camila cabello", tweet)
 tweet = re.sub(r"RondaRousey", "Ronda Rousey", tweet)
 tweet = re.sub(r"MTVHottest", "MTV Hottest", tweet)
 tweet = re.sub(r"TrapMusic", "Trap Music", tweet)
 tweet = re.sub(r"ProphetMuhammad", "Prophet Muhammad", tweet)
 tweet = re.sub(r"PantherAttack", "Panther Attack", tweet)
 tweet = re.sub(r"StrategicPatience", "Strategic Patience", tweet)
 tweet = re.sub(r"socialnews", "social news", tweet)
 tweet = re.sub(r"IDPs:", "Internally Displaced People :", tweet)
 tweet = re.sub(r"ArtistsUnited", "Artists United", tweet)
 tweet = re.sub(r"ClaytonBryant", "Clayton Bryant", tweet)
 tweet = re.sub(r"jimmyfallon", "jimmy fallon", tweet)
 tweet = re.sub(r"justinbieber", "justin bieber", tweet)
 tweet = re.sub(r"Time2015", "Time 2015", tweet)
 tweet = re.sub(r"djicemoon", "dj icemoon", tweet)
 tweet = re.sub(r"LivingSafely", "Living Safely", tweet)
 tweet = re.sub(r"FIFA16", "Fifa 2016", tweet)
 tweet = re.sub(r"thisiswhywecanthavenicethings", "this is why we cannot have nice things", tweet)
 tweet = re.sub(r"bbcnews", "bbc news", tweet)
 tweet = re.sub(r"UndergroundRailraod", "Underground Railraod", tweet)
 tweet = re.sub(r"c4news", "c4 news", tweet)
 tweet = re.sub(r"MUDSLIDE", "mudslide", tweet)
 tweet = re.sub(r"NoSurrender", "No Surrender", tweet)
 tweet = re.sub(r"NotExplained", "Not Explained", tweet)
 tweet = re.sub(r"greatbritishbakeoff", "great british bake off", tweet)
 tweet = re.sub(r"LondonFire", "London Fire", tweet)
 tweet = re.sub(r"KOTAWeather", "KOTA Weather", tweet)
 tweet = re.sub(r"LuchaUnderground", "Lucha Underground", tweet)
 tweet = re.sub(r"KOIN6News", "KOIN 6 News", tweet)
 tweet = re.sub(r"LiveOnK2", "Live On K2", tweet)
 tweet = re.sub(r"9NewsGoldCoast", "9 News Gold Coast", tweet)
 tweet = re.sub(r"nikeplus", "nike plus", tweet)
 tweet = re.sub(r"david_cameron", "David Cameron", tweet)
 tweet = re.sub(r"peterjukes", "Peter Jukes", tweet)
 tweet = re.sub(r"MikeParrActor", "Michael Parr", tweet)
 tweet = re.sub(r"4PlayThursdays", "Foreplay Thursdays", tweet)
 tweet = re.sub(r"TGF2015", "Tontitown Grape Festival", tweet)
 tweet = re.sub(r"realmandyrain", "Mandy Rain", tweet)
 tweet = re.sub(r"GraysonDolan", "Grayson Dolan", tweet)
 tweet = re.sub(r"ApolloBrown", "Apollo Brown", tweet)
 tweet = re.sub(r"saddlebrooke", "Saddlebrooke", tweet)
 tweet = re.sub(r"TontitownGrape", "Tontitown Grape", tweet)
 tweet = re.sub(r"AbbsWinston", "Abbs Winston", tweet)
 tweet = re.sub(r"ShaunKing", "Shaun King", tweet)
 tweet = re.sub(r"MeekMill", "Meek Mill", tweet)
 tweet = re.sub(r"TornadoGiveaway", "Tornado Giveaway", tweet)
 tweet = re.sub(r"GRupdates", "GR updates", tweet)
 tweet = re.sub(r"SouthDowns", "South Downs", tweet)
 tweet = re.sub(r"braininjury", "brain injury", tweet)
 tweet = re.sub(r"auspol", "Australian politics", tweet)
 tweet = re.sub(r"PlannedParenthood", "Planned Parenthood", tweet)
 tweet = re.sub(r"calgaryweather", "Calgary Weather", tweet)
 tweet = re.sub(r"weallheartonedirection", "we all heart one direction", tweet)
 tweet = re.sub(r"edsheeran", "Ed Sheeran", tweet)
 tweet = re.sub(r"TrueHeroes", "True Heroes", tweet)
 tweet = re.sub(r"ComplexMag", "Complex Magazine", tweet)
 tweet = re.sub(r"TheAdvocateMag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"CityofCalgary", "City of Calgary", tweet)
 tweet = re.sub(r"EbolaOutbreak", "Ebola Outbreak", tweet)
 tweet = re.sub(r"SummerFate", "Summer Fate", tweet)
 tweet = re.sub(r"RAmag", "Royal Academy Magazine", tweet)
 tweet = re.sub(r"offers2go", "offers to go", tweet)
 tweet = re.sub(r"ModiMinistry", "Modi Ministry", tweet)
 tweet = re.sub(r"TAXIWAYS", "taxi ways", tweet)
 tweet = re.sub(r"Calum5SOS", "Calum Hood", tweet)
 tweet = re.sub(r"JamesMelville", "James Melville", tweet)
 tweet = re.sub(r"JamaicaObserver", "Jamaica Observer", tweet)
 tweet = re.sub(r"TweetLikeItsSeptember11th2001", "Tweet like it is september 11th 2001", tweet)
 tweet = re.sub(r"cbplawyers", "cbp lawyers", tweet)
 tweet = re.sub(r"fewmoretweets", "few more tweets", tweet)
 tweet = re.sub(r"BlackLivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"NASAHurricane", "NASA Hurricane", tweet)
 tweet = re.sub(r"onlinecommunities", "online communities", tweet)
 tweet = re.sub(r"humanconsumption", "human consumption", tweet)
 tweet = re.sub(r"Typhoon-Devastated", "Typhoon Devastated", tweet)
 tweet = re.sub(r"Meat-Loving", "Meat Loving", tweet)
 tweet = re.sub(r"facialabuse", "facial abuse", tweet)
 tweet = re.sub(r"LakeCounty", "Lake County", tweet)
 tweet = re.sub(r"BeingAuthor", "Being Author", tweet)
 tweet = re.sub(r"withheavenly", "with heavenly", tweet)
 tweet = re.sub(r"thankU", "thank you", tweet)
 tweet = re.sub(r"iTunesMusic", "iTunes Music", tweet)
 tweet = re.sub(r"OffensiveContent", "Offensive Content", tweet)
 tweet = re.sub(r"WorstSummerJob", "Worst Summer Job", tweet)
 tweet = re.sub(r"HarryBeCareful", "Harry Be Careful", tweet)
 tweet = re.sub(r"NASASolarSystem", "NASA Solar System", tweet)
 tweet = re.sub(r"animalrescue", "animal rescue", tweet)
 tweet = re.sub(r"KurtSchlichter", "Kurt Schlichter", tweet)
 tweet = re.sub(r"Throwingknifes", "Throwing knives", tweet)
 tweet = re.sub(r"GodsLove", "God's Love", tweet)
 tweet = re.sub(r"bookboost", "book boost", tweet)
 tweet = re.sub(r"ibooklove", "I book love", tweet)
 tweet = re.sub(r"NestleIndia", "Nestle India", tweet)
 tweet = re.sub(r"realDonaldTrump", "Donald Trump", tweet)
 tweet = re.sub(r"DavidVonderhaar", "David Vonderhaar", tweet)
 tweet = re.sub(r"CecilTheLion", "Cecil The Lion", tweet)
 tweet = re.sub(r"weathernetwork", "weather network", tweet)
 tweet = re.sub(r"GOPDebate", "GOP Debate", tweet)
 tweet = re.sub(r"RickPerry", "Rick Perry", tweet)
 tweet = re.sub(r"frontpage", "front page", tweet)
 tweet = re.sub(r"NewsInTweets", "News In Tweets", tweet)
 tweet = re.sub(r"ViralSpell", "Viral Spell", tweet)
 tweet = re.sub(r"til_now", "until now", tweet)
 tweet = re.sub(r"volcanoinRussia", "volcano in Russia", tweet)
 tweet = re.sub(r"ZippedNews", "Zipped News", tweet)
 tweet = re.sub(r"MicheleBachman", "Michele Bachman", tweet)
 tweet = re.sub(r"53inch", "53 inch", tweet)
 tweet = re.sub(r"KerrickTrial", "Kerrick Trial", tweet)
 tweet = re.sub(r"abstorm", "Alberta Storm", tweet)
 tweet = re.sub(r"Beyhive", "Beyonce hive", tweet)
 tweet = re.sub(r"RockyFire", "Rocky Fire", tweet)
 tweet = re.sub(r"Listen/Buy", "Listen / Buy", tweet)
 tweet = re.sub(r"ArtistsUnited", "Artists United", tweet)
 tweet = re.sub(r"ENGvAUS", "England vs Australia", tweet)
 tweet = re.sub(r"ScottWalker", "Scott Walker", tweet)
 return tweet
total['text'] = total['text'].apply(clean )<define_variables>
ntrain = train.shape[0]
ntest = test.shape[0]
SEED = 0
NFOLDS = 5
kf = KFold(ntrain, n_folds= NFOLDS, random_state=SEED)
 class SklearnHelper(object):
def __init__(self, clf, seed=0, params=None):
params['random_state'] = seed
self.clf = clf(**params)
 def train(self, x_train, y_train):
self.clf.fit(x_train, y_train)
 def predict(self, x):
return self.clf.predict(x)
 def fit(self,x,y):
return self.clf.fit(x,y)
 def feature_importances(self,x,y):
print(self.clf.fit(x,y ).feature_importances_)
tweets = [tweet for tweet in total['text']]
train = total[:len(train)]
test = total[len(train):]<categorify>
def get_oof(clf, x_train, y_train, x_test):
oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
oof_test_skf = np.empty(( NFOLDS, ntest))
for i,(train_index, test_index)in enumerate(kf):
x_tr = x_train[train_index]
y_tr = y_train[train_index]
x_te = x_train[test_index]
clf.train(x_tr, y_tr)
 oof_train[test_index] = clf.predict(x_te)
 oof_test_skf[i, :] = clf.predict(x_test)
 oof_test[:] = oof_test_skf.mean(axis=0)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1 )
def generate_ngrams(text, n_gram=1):
token = [token for token in text.lower().split(' ')if token != '' if token not in wordcloud.STOPWORDS]
ngrams = zip(*[token[i:] for i in range(n_gram)])
 return [' '.join(ngram)for ngram in ngrams]
disaster_unigrams = defaultdict(int)
 for word in total[train['target'] == 1]['text']:
for word in generate_ngrams(word, n_gram = 1):
disaster_unigrams[word] += 1
disaster_unigrams = pd.DataFrame(sorted(disaster_unigrams.items() , key=lambda x: x[1])[::-1])
 nondisaster_unigrams = defaultdict(int)
 for word in total[train['target'] == 0]['text']:
for word in generate_ngrams(word, n_gram = 1):
nondisaster_unigrams[word] += 1
nondisaster_unigrams = pd.DataFrame(sorted(nondisaster_unigrams.items() , key=lambda x: x[1])[::-1])
 disaster_bigrams = defaultdict(int)
 for word in total[train['target'] == 1]['text']:
for word in generate_ngrams(word, n_gram = 2):
disaster_bigrams[word] += 1
disaster_bigrams = pd.DataFrame(sorted(disaster_bigrams.items() , key=lambda x: x[1])[::-1])
 nondisaster_bigrams = defaultdict(int)
 for word in total[train['target'] == 0]['text']:
for word in generate_ngrams(word, n_gram = 2):
nondisaster_bigrams[word] += 1
nondisaster_bigrams = pd.DataFrame(sorted(nondisaster_bigrams.items() , key=lambda x: x[1])[::-1])
 disaster_trigrams = defaultdict(int)
 for word in total[train['target'] == 1]['text']:
for word in generate_ngrams(word, n_gram = 3):
disaster_trigrams[word] += 1
disaster_trigrams = pd.DataFrame(sorted(disaster_trigrams.items() , key=lambda x: x[1])[::-1])
 nondisaster_trigrams = defaultdict(int)
 for word in total[train['target'] == 0]['text']:
for word in generate_ngrams(word, n_gram = 3):
nondisaster_trigrams[word] += 1
nondisaster_trigrams = pd.DataFrame(sorted(nondisaster_trigrams.items() , key=lambda x: x[1])[::-1])
 disaster_4grams = defaultdict(int)
 for word in total[train['target'] == 1]['text']:
for word in generate_ngrams(word, n_gram = 4):
disaster_4grams[word] += 1
disaster_4grams = pd.DataFrame(sorted(disaster_4grams.items() , key=lambda x: x[1])[::-1])
 nondisaster_4grams = defaultdict(int)
 for word in total[train['target'] == 0]['text']:
for word in generate_ngrams(word, n_gram = 4):
nondisaster_4grams[word] += 1
nondisaster_4grams = pd.DataFrame(sorted(nondisaster_4grams.items() , key=lambda x: x[1])[::-1] )<string_transform>
rf_params = {
'n_jobs': -1,
'n_estimators': 500,
'warm_start': True,
'max_depth': 6,
'min_samples_leaf': 2,
'max_features' : 'sqrt',
'verbose': 0
}
et_params = {
'n_jobs': -1,
'n_estimators':500,
'max_depth': 8,
'min_samples_leaf': 2,
'verbose': 0
}
ada_params = {
'n_estimators': 500,
'learning_rate' : 0.75
}
gb_params = {
'n_estimators': 500,
'max_depth': 5,
'min_samples_leaf': 2,
'verbose': 0
}
svc_params = {
'kernel' : 'linear',
'C' : 0.025
}
to_exclude = '*+-/() %
[\\]{|}^_`~\t'
to_tokenize = '!"
tokenizer = Tokenizer(filters = to_exclude)
 text = 'Why are you so f%
text = re.sub(r'(['+to_tokenize+'])', r' \1 ', text)
 tokenizer.fit_on_texts([text])
 print(tokenizer.word_index )<feature_engineering>
rf = SklearnHelper(clf=RandomForestClassifier, seed=SEED, params=rf_params)
 et = SklearnHelper(clf=ExtraTreesClassifier, seed=SEED, params=et_params)
 ada = SklearnHelper(clf=AdaBoostClassifier, seed=SEED, params=ada_params)
 gb = SklearnHelper(clf=GradientBoostingClassifier, seed=SEED, params=gb_params)
 svc = SklearnHelper(clf=SVC, seed=SEED, params=svc_params )
<string_transform>
y_train = train['Survived'].ravel()
train = train.drop(['Survived'], axis=1)
 x_train = train.values
x_test = test.values
tokenizer = Tokenizer()
tokenizer.fit_on_texts(tweets)
 sequences = tokenizer.texts_to_sequences(tweets)
 word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
data = pad_sequences(sequences)
 labels = train['target']
print('Shape of data tensor:', data.shape)
 print('Shape of label tensor:', labels.shape)
 nlp_train = data[:len(train)]
labels = labels
nlp_test = data[len(train):]
MAX_SEQUENCE_LENGTH = data.shape[1]<feature_engineering>
et_oof_train, et_oof_test = get_oof(et, x_train, y_train, x_test)
 rf_oof_train, rf_oof_test = get_oof(rf,x_train, y_train, x_test)
 ada_oof_train, ada_oof_test = get_oof(ada, x_train, y_train, x_test)
 gb_oof_train, gb_oof_test = get_oof(gb,x_train, y_train, x_test)
 svc_oof_train, svc_oof_test = get_oof(svc,x_train, y_train, x_test)
 print("Training is complete" )
embeddings_index = {}
with open('.. /input/glove-global-vectors-for-word-representation/glove.6B.200d.txt','r')as f:
for line in tqdm(f):
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
print('Found %s word vectors in the GloVe library' % len(embeddings_index))<define_variables>
rf_feature = rf.feature_importances(x_train,y_train)
 et_feature = et.feature_importances(x_train, y_train)
 ada_feature = ada.feature_importances(x_train, y_train)
 gb_feature = gb.feature_importances(x_train,y_train )
EMBEDDING_DIM = 200<categorify>
rf_features = [0.10474135, 0.21837029, 0.04432652, 0.02249159, 0.05432591, 0.02854371
,0.07570305, 0.01088129 , 0.24247496, 0.13685733 , 0.06128402]
et_features = [ 0.12165657, 0.37098307 ,0.03129623 , 0.01591611 , 0.05525811 , 0.028157
,0.04589793 , 0.02030357 , 0.17289562 , 0.04853517, 0.08910063]
ada_features = [0.028 , 0.008 , 0.012 , 0.05866667, 0.032 , 0.008
,0.04666667 , 0., 0.05733333, 0.73866667, 0.01066667]
gb_features = [ 0.06796144 , 0.03889349 , 0.07237845 , 0.02628645 , 0.11194395, 0.04778854
,0.05965792 , 0.02774745, 0.07462718, 0.4593142 , 0.01340093]
embedding_matrix = np.zeros(( len(word_index)+ 1, EMBEDDING_DIM))
for word, i in tqdm(word_index.items()):
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
print("Our embedded matrix is of dimension", embedding_matrix.shape )<choose_model_class>
cols = train.columns.values
feature_dataframe = pd.DataFrame({'features': cols,
'Random Forest feature importances': rf_features,
'Extra Trees feature importances': et_features,
'AdaBoost feature importances': ada_features,
'Gradient Boost feature importances': gb_features
} )
embedding = Embedding(len(word_index)+ 1, EMBEDDING_DIM, weights = [embedding_matrix],
input_length = MAX_SEQUENCE_LENGTH, trainable = False)
<normalization>
feature_dataframe['mean'] = feature_dataframe.mean(axis= 1)
 feature_dataframe.head(3 )
def scale(df, scaler):
return scaler.fit_transform(df.iloc[:, 2:])
 meta_train = scale(train, StandardScaler())
 meta_test = scale(test, StandardScaler() )<choose_model_class>
base_predictions_train = pd.DataFrame({'RandomForest': rf_oof_train.ravel() ,
'ExtraTrees': et_oof_train.ravel() ,
'AdaBoost': ada_oof_train.ravel() ,
'GradientBoost': gb_oof_train.ravel()
})
 base_predictions_train.head()
def create_lstm(spatial_dropout, dropout, recurrent_dropout, learning_rate, bidirectional = False):
activation = LeakyReLU(alpha = 0.01)
 nlp_input = Input(shape =(MAX_SEQUENCE_LENGTH,), name = 'nlp_input')
 meta_input_train = Input(shape =(7,), name = 'meta_train')
 emb = embedding(nlp_input)
 emb = SpatialDropout1D(dropout )(emb)
 if bidirectional:
nlp_out =(Bidirectional(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal')) )(emb)
 else:
nlp_out =(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal'))(emb)
 x = Concatenate()([nlp_out, meta_input_train])
 x = Dropout(dropout )(x)
 preds = Dense(1, activation='sigmoid', kernel_regularizer = regularizers.l2(1e-4))(x)
 model = Model(inputs=[nlp_input , meta_input_train], outputs = preds)
 optimizer = Adam(learning_rate = learning_rate)
 model.compile(loss = 'binary_crossentropy', optimizer = optimizer, metrics = ['accuracy'])
 return model<choose_model_class>
x_train = np.concatenate(( et_oof_train, rf_oof_train, ada_oof_train, gb_oof_train, svc_oof_train), axis=1)
 x_test = np.concatenate(( et_oof_test, rf_oof_test, ada_oof_test, gb_oof_test, svc_oof_test), axis=1 )
lstm = create_lstm(spatial_dropout =.2, dropout =.2, recurrent_dropout =.2,
learning_rate = 3e-4, bidirectional = True)
 lstm.summary()<train_model>
gbm = xgb.XGBClassifier(
 n_estimators= 2000,
max_depth= 4,
min_child_weight= 2,
gamma=0.9,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread= -1,
scale_pos_weight=1 ).fit(x_train, y_train)
 predictions = gbm.predict(x_test )
<choose_model_class><EOS> 
StackingSubmission = pd.DataFrame({ 'PassengerId': PassengerId,
'Survived': predictions })
 StackingSubmission.to_csv("gender_submission.csv", index=False )