Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<prepare_x_and_y>
plt.style.use('ggplot')
 warnings.filterwarnings('ignore')
y = train['target']<prepare_x_and_y>
def seed_everything(seed):
os.environ['PYTHONHASHSEED']=str(seed)
 tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 seed_everything(34 )
test_x = test['text']<split>
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 train.head()
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state = rand_state, shuffle = True )<feature_engineering>
test_id = test['id']
columns = {'id', 'location'}
train = train.drop(columns = columns)
 test = test.drop(columns = columns)
 train['keyword'] = train['keyword'].fillna('unknown')
 test['keyword'] = test['keyword'].fillna('unknown')
 train['text'] = train['text'] + ' ' + train['keyword']
test['text'] = test['text'] + ' ' + test['keyword']
columns = {'keyword'}
train = train.drop(columns = columns)
 test = test.drop(columns = columns)
 total = train.append(test )
count_vectorizer = CountVectorizer(stop_words='english')
 count_train = count_vectorizer.fit_transform(X_train)
 count_test = count_vectorizer.transform(X_test)
 count_train_sub = count_vectorizer.transform(X)
 count_sub = count_vectorizer.transform(test_x)
<compute_train_metric>
total['unique word count'] = total['text'].apply(lambda x: len(set(x.split())))
 total['stopword count'] = total['text'].apply(lambda x: len([i for i in x.lower().split() if i in wordcloud.STOPWORDS]))
total['stopword ratio'] = total['stopword count'] / total['word count']
total['punctuation count'] = total['text'].apply(lambda x: len([i for i in str(x)if i in string.punctuation]))
train = total[:len(train)]
disaster = train['target'] == 1
fig, axes = plt.subplots(4, figsize=(20, 30))
graph1 = sns.kdeplot(train.loc[~disaster]['unique word count'], shade = True, label = 'Not Disaster', ax=axes[0])
 graph1 = sns.kdeplot(train.loc[disaster]['unique word count'], shade = True, label = 'Disaster', ax=axes[0])
 graph1.set_title('Distribution of Unique Word Count')
 graph2 = sns.kdeplot(train.loc[~disaster]['stopword count'], shade = True, label = 'Not Disaster', ax=axes[1])
 graph2 = sns.kdeplot(train.loc[disaster]['stopword count'], shade = True, label = 'Disaster', ax=axes[1])
 graph2.set_title('Distribution of Stopword Word Count')
 graph3 = sns.kdeplot(train.loc[~disaster]['punctuation count'], shade = True, label = 'Not Disaster', ax=axes[2], bw = 1)
 graph3 = sns.kdeplot(train.loc[disaster]['punctuation count'], shade = True, label = 'Disaster', ax=axes[2], bw = 1)
 graph3.set_title('Distribution of Punctuation Count')
 graph4 = sns.kdeplot(train.loc[~disaster]['stopword ratio'], shade = True, label = 'Not Disaster', ax=axes[3], bw =.05)
 graph4 = sns.kdeplot(train.loc[disaster]['stopword ratio'], shade = True, label = 'Disaster', ax=axes[3], bw =.05)
 graph4.set_title('Distribution of Stopword Ratio')
 fig.tight_layout()
plt.show()
count_nb = MultinomialNB()
count_nb.fit(count_train ,y_train)
 count_nb_pred = count_nb.predict(count_test)
 count_nb_score = accuracy_score(y_test,count_nb_pred)
 print('MultinomialNaiveBayes Count Score: ', count_nb_score)
 count_nb_cm = confusion_matrix(y_test, count_nb_pred)
 count_nb_cm<compute_train_metric>
def remove_punctuation(x):
return x.translate(str.maketrans('', '', string.punctuation))
def remove_stopwords(x):
return ' '.join([i for i in x.split() if i not in wordcloud.STOPWORDS])
 def remove_less_than(x):
return ' '.join([i for i in x.split() if len(i)> 3])
 def remove_non_alphabet(x):
return ' '.join([i for i in x.split() if i.isalpha() ])
 def strip_all_entities(x):
return ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)"," ",x ).split() )
count_bnb = BernoulliNB()
count_bnb.fit(count_train ,y_train)
 count_bnb_pred = count_bnb.predict(count_test)
 count_bnb_score = accuracy_score(y_test,count_bnb_pred)
 print('BernoulliNaiveBayes Count Score: ', count_bnb_score)
 count_bnb_cm = confusion_matrix(y_test, count_bnb_pred)
 count_bnb_cm<compute_train_metric>
strip_all_entities('@shawn Titanic
Times: Telegraph.co.ukTitanic tragedy could have been preve...http://bet.ly/tuN2wx' )
count_lsvc = LinearSVC()
count_lsvc.fit(count_train ,y_train)
 count_lsvc_pred = count_lsvc.predict(count_test)
 count_lsvc_score = accuracy_score(y_test,count_lsvc_pred)
 print('LinearSVC Count Score: ', count_lsvc_score)
 count_lsvc_cm = confusion_matrix(y_test, count_lsvc_pred)
 count_lsvc_cm<compute_train_metric>
!pip install autocorrect
def spell_check(x):
spell = Speller(lang='en')
 return " ".join([spell(i)for i in x.split() ])
 mispelled = 'Pleaze spelcheck this sentince'
spell_check(mispelled )
count_svc = SVC()
count_svc.fit(count_train ,y_train)
 count_svc_pred = count_svc.predict(count_test)
 count_svc_score = accuracy_score(y_test,count_svc_pred)
 print('SVC Count Score: ', count_svc_score)
 count_svc_cm = confusion_matrix(y_test, count_svc_pred)
 count_svc_cm<compute_train_metric>
PROCESS_TWEETS = False
if PROCESS_TWEETS:
total['text'] = total['text'].apply(lambda x: x.lower())
 total['text'] = total['text'].apply(lambda x: re.sub(r'https?://\S+|www\.\S+', '', x, flags = re.MULTILINE))
total['text'] = total['text'].apply(remove_punctuation)
 total['text'] = total['text'].apply(remove_stopwords)
 total['text'] = total['text'].apply(remove_less_than)
 total['text'] = total['text'].apply(remove_non_alphabet)
 total['text'] = total['text'].apply(spell_check )
count_nusvc = NuSVC(0.4)
 count_nusvc.fit(count_train ,y_train)
 count_nusvc_pred = count_nusvc.predict(count_test)
 count_nusvc_score = accuracy_score(y_test,count_nusvc_pred)
 print('NuSVC Count Score: ', count_nusvc_score)
 count_nusvc_cm = confusion_matrix(y_test, count_nusvc_pred)
 count_nusvc_cm<train_on_grid>
contractions = {
"ain't": "am not / are not / is not / has not / have not",
"aren't": "are not / am not",
"can't": "cannot",
"can't've": "cannot have",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"couldn't've": "could not have",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hadn't've": "had not have",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he had / he would",
"he'd've": "he would have",
"he'll": "he shall / he will",
"he'll've": "he shall have / he will have",
"he's": "he has / he is",
"how'd": "how did",
"how'd'y": "how do you",
"how'll": "how will",
"how's": "how has / how is / how does",
"I'd": "I had / I would",
"I'd've": "I would have",
"I'll": "I shall / I will",
"I'll've": "I shall have / I will have",
"I'm": "I am",
"I've": "I have",
"isn't": "is not",
"it'd": "it had / it would",
"it'd've": "it would have",
"it'll": "it shall / it will",
"it'll've": "it shall have / it will have",
"it's": "it has / it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"mightn't've": "might not have",
"must've": "must have",
"mustn't": "must not",
"mustn't've": "must not have",
"needn't": "need not",
"needn't've": "need not have",
"o'clock": "of the clock",
"oughtn't": "ought not",
"oughtn't've": "ought not have",
"shan't": "shall not",
"sha'n't": "shall not",
"shan't've": "shall not have",
"she'd": "she had / she would",
"she'd've": "she would have",
"she'll": "she shall / she will",
"she'll've": "she shall have / she will have",
"she's": "she has / she is",
"should've": "should have",
"shouldn't": "should not",
"shouldn't've": "should not have",
"so've": "so have",
"so's": "so as / so is",
"that'd": "that would / that had",
"that'd've": "that would have",
"that's": "that has / that is",
"there'd": "there had / there would",
"there'd've": "there would have",
"there's": "there has / there is",
"they'd": "they had / they would",
"they'd've": "they would have",
"they'll": "they shall / they will",
"they'll've": "they shall have / they will have",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd": "we had / we would",
"we'd've": "we would have",
"we'll": "we will",
"we'll've": "we will have",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what shall / what will",
"what'll've": "what shall have / what will have",
"what're": "what are",
"what's": "what has / what is",
"what've": "what have",
"when's": "when has / when is",
"when've": "when have",
"where'd": "where did",
"where's": "where has / where is",
"where've": "where have",
"who'll": "who shall / who will",
"who'll've": "who shall have / who will have",
"who's": "who has / who is",
"who've": "who have",
"why's": "why has / why is",
"why've": "why have",
"will've": "will have",
"won't": "will not",
"won't've": "will not have",
"would've": "would have",
"wouldn't": "would not",
"wouldn't've": "would not have",
"y'all": "you all",
"y'all'd": "you all would",
"y'all'd've": "you all would have",
"y'all're": "you all are",
"y'all've": "you all have",
"you'd": "you had / you would",
"you'd've": "you would have",
"you'll": "you shall / you will",
"you'll've": "you shall have / you will have",
"you're": "you are",
"you've": "you have"
}
contractions_re = re.compile('(%s)' % '|'.join(contractions.keys()))
def expand_contractions(s, contractions = contractions):
def replace(match):
return contractions[match.group(0)]
return contractions_re.sub(replace, s)
 expand_contractions("can't stop won't stop" )
<compute_test_metric>
total['text'] = total['text'].apply(expand_contractions )
<compute_train_metric>
def clean(tweet):
tweet = re.sub(r"tnwx", "Tennessee Weather", tweet)
 tweet = re.sub(r"azwx", "Arizona Weather", tweet)
 tweet = re.sub(r"alwx", "Alabama Weather", tweet)
 tweet = re.sub(r"wordpressdotcom", "wordpress", tweet)
 tweet = re.sub(r"gawx", "Georgia Weather", tweet)
 tweet = re.sub(r"scwx", "South Carolina Weather", tweet)
 tweet = re.sub(r"cawx", "California Weather", tweet)
 tweet = re.sub(r"usNWSgov", "United States National Weather Service", tweet)
 tweet = re.sub(r"MH370", "Malaysia Airlines Flight 370", tweet)
 tweet = re.sub(r"okwx", "Oklahoma City Weather", tweet)
 tweet = re.sub(r"arwx", "Arkansas Weather", tweet)
 tweet = re.sub(r"lmao", "laughing my ass off", tweet)
 tweet = re.sub(r"amirite", "am I right", tweet)
 tweet = re.sub(r"w/e", "whatever", tweet)
 tweet = re.sub(r"w/", "with", tweet)
 tweet = re.sub(r"USAgov", "USA government", tweet)
 tweet = re.sub(r"recentlu", "recently", tweet)
 tweet = re.sub(r"Ph0tos", "Photos", tweet)
 tweet = re.sub(r"exp0sed", "exposed", tweet)
 tweet = re.sub(r"<3", "love", tweet)
 tweet = re.sub(r"amageddon", "armageddon", tweet)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"WindStorm", "Wind Storm", tweet)
 tweet = re.sub(r"16yr", "16 year", tweet)
 tweet = re.sub(r"TRAUMATISED", "traumatized", tweet)
 tweet = re.sub(r"IranDeal", "Iran Deal", tweet)
 tweet = re.sub(r"ArianaGrande", "Ariana Grande", tweet)
 tweet = re.sub(r"camilacabello97", "camila cabello", tweet)
 tweet = re.sub(r"RondaRousey", "Ronda Rousey", tweet)
 tweet = re.sub(r"MTVHottest", "MTV Hottest", tweet)
 tweet = re.sub(r"TrapMusic", "Trap Music", tweet)
 tweet = re.sub(r"ProphetMuhammad", "Prophet Muhammad", tweet)
 tweet = re.sub(r"PantherAttack", "Panther Attack", tweet)
 tweet = re.sub(r"StrategicPatience", "Strategic Patience", tweet)
 tweet = re.sub(r"socialnews", "social news", tweet)
 tweet = re.sub(r"IDPs:", "Internally Displaced People :", tweet)
 tweet = re.sub(r"ArtistsUnited", "Artists United", tweet)
 tweet = re.sub(r"ClaytonBryant", "Clayton Bryant", tweet)
 tweet = re.sub(r"jimmyfallon", "jimmy fallon", tweet)
 tweet = re.sub(r"justinbieber", "justin bieber", tweet)
 tweet = re.sub(r"Time2015", "Time 2015", tweet)
 tweet = re.sub(r"djicemoon", "dj icemoon", tweet)
 tweet = re.sub(r"LivingSafely", "Living Safely", tweet)
 tweet = re.sub(r"FIFA16", "Fifa 2016", tweet)
 tweet = re.sub(r"thisiswhywecanthavenicethings", "this is why we cannot have nice things", tweet)
 tweet = re.sub(r"bbcnews", "bbc news", tweet)
 tweet = re.sub(r"UndergroundRailraod", "Underground Railraod", tweet)
 tweet = re.sub(r"c4news", "c4 news", tweet)
 tweet = re.sub(r"MUDSLIDE", "mudslide", tweet)
 tweet = re.sub(r"NoSurrender", "No Surrender", tweet)
 tweet = re.sub(r"NotExplained", "Not Explained", tweet)
 tweet = re.sub(r"greatbritishbakeoff", "great british bake off", tweet)
 tweet = re.sub(r"LondonFire", "London Fire", tweet)
 tweet = re.sub(r"KOTAWeather", "KOTA Weather", tweet)
 tweet = re.sub(r"LuchaUnderground", "Lucha Underground", tweet)
 tweet = re.sub(r"KOIN6News", "KOIN 6 News", tweet)
 tweet = re.sub(r"LiveOnK2", "Live On K2", tweet)
 tweet = re.sub(r"9NewsGoldCoast", "9 News Gold Coast", tweet)
 tweet = re.sub(r"nikeplus", "nike plus", tweet)
 tweet = re.sub(r"david_cameron", "David Cameron", tweet)
 tweet = re.sub(r"peterjukes", "Peter Jukes", tweet)
 tweet = re.sub(r"MikeParrActor", "Michael Parr", tweet)
 tweet = re.sub(r"4PlayThursdays", "Foreplay Thursdays", tweet)
 tweet = re.sub(r"TGF2015", "Tontitown Grape Festival", tweet)
 tweet = re.sub(r"realmandyrain", "Mandy Rain", tweet)
 tweet = re.sub(r"GraysonDolan", "Grayson Dolan", tweet)
 tweet = re.sub(r"ApolloBrown", "Apollo Brown", tweet)
 tweet = re.sub(r"saddlebrooke", "Saddlebrooke", tweet)
 tweet = re.sub(r"TontitownGrape", "Tontitown Grape", tweet)
 tweet = re.sub(r"AbbsWinston", "Abbs Winston", tweet)
 tweet = re.sub(r"ShaunKing", "Shaun King", tweet)
 tweet = re.sub(r"MeekMill", "Meek Mill", tweet)
 tweet = re.sub(r"TornadoGiveaway", "Tornado Giveaway", tweet)
 tweet = re.sub(r"GRupdates", "GR updates", tweet)
 tweet = re.sub(r"SouthDowns", "South Downs", tweet)
 tweet = re.sub(r"braininjury", "brain injury", tweet)
 tweet = re.sub(r"auspol", "Australian politics", tweet)
 tweet = re.sub(r"PlannedParenthood", "Planned Parenthood", tweet)
 tweet = re.sub(r"calgaryweather", "Calgary Weather", tweet)
 tweet = re.sub(r"weallheartonedirection", "we all heart one direction", tweet)
 tweet = re.sub(r"edsheeran", "Ed Sheeran", tweet)
 tweet = re.sub(r"TrueHeroes", "True Heroes", tweet)
 tweet = re.sub(r"ComplexMag", "Complex Magazine", tweet)
 tweet = re.sub(r"TheAdvocateMag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"CityofCalgary", "City of Calgary", tweet)
 tweet = re.sub(r"EbolaOutbreak", "Ebola Outbreak", tweet)
 tweet = re.sub(r"SummerFate", "Summer Fate", tweet)
 tweet = re.sub(r"RAmag", "Royal Academy Magazine", tweet)
 tweet = re.sub(r"offers2go", "offers to go", tweet)
 tweet = re.sub(r"ModiMinistry", "Modi Ministry", tweet)
 tweet = re.sub(r"TAXIWAYS", "taxi ways", tweet)
 tweet = re.sub(r"Calum5SOS", "Calum Hood", tweet)
 tweet = re.sub(r"JamesMelville", "James Melville", tweet)
 tweet = re.sub(r"JamaicaObserver", "Jamaica Observer", tweet)
 tweet = re.sub(r"TweetLikeItsSeptember11th2001", "Tweet like it is september 11th 2001", tweet)
 tweet = re.sub(r"cbplawyers", "cbp lawyers", tweet)
 tweet = re.sub(r"fewmoretweets", "few more tweets", tweet)
 tweet = re.sub(r"BlackLivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"NASAHurricane", "NASA Hurricane", tweet)
 tweet = re.sub(r"onlinecommunities", "online communities", tweet)
 tweet = re.sub(r"humanconsumption", "human consumption", tweet)
 tweet = re.sub(r"Typhoon-Devastated", "Typhoon Devastated", tweet)
 tweet = re.sub(r"Meat-Loving", "Meat Loving", tweet)
 tweet = re.sub(r"facialabuse", "facial abuse", tweet)
 tweet = re.sub(r"LakeCounty", "Lake County", tweet)
 tweet = re.sub(r"BeingAuthor", "Being Author", tweet)
 tweet = re.sub(r"withheavenly", "with heavenly", tweet)
 tweet = re.sub(r"thankU", "thank you", tweet)
 tweet = re.sub(r"iTunesMusic", "iTunes Music", tweet)
 tweet = re.sub(r"OffensiveContent", "Offensive Content", tweet)
 tweet = re.sub(r"WorstSummerJob", "Worst Summer Job", tweet)
 tweet = re.sub(r"HarryBeCareful", "Harry Be Careful", tweet)
 tweet = re.sub(r"NASASolarSystem", "NASA Solar System", tweet)
 tweet = re.sub(r"animalrescue", "animal rescue", tweet)
 tweet = re.sub(r"KurtSchlichter", "Kurt Schlichter", tweet)
 tweet = re.sub(r"Throwingknifes", "Throwing knives", tweet)
 tweet = re.sub(r"GodsLove", "God's Love", tweet)
 tweet = re.sub(r"bookboost", "book boost", tweet)
 tweet = re.sub(r"ibooklove", "I book love", tweet)
 tweet = re.sub(r"NestleIndia", "Nestle India", tweet)
 tweet = re.sub(r"realDonaldTrump", "Donald Trump", tweet)
 tweet = re.sub(r"DavidVonderhaar", "David Vonderhaar", tweet)
 tweet = re.sub(r"CecilTheLion", "Cecil The Lion", tweet)
 tweet = re.sub(r"weathernetwork", "weather network", tweet)
 tweet = re.sub(r"GOPDebate", "GOP Debate", tweet)
 tweet = re.sub(r"RickPerry", "Rick Perry", tweet)
 tweet = re.sub(r"frontpage", "front page", tweet)
 tweet = re.sub(r"NewsInTweets", "News In Tweets", tweet)
 tweet = re.sub(r"ViralSpell", "Viral Spell", tweet)
 tweet = re.sub(r"til_now", "until now", tweet)
 tweet = re.sub(r"volcanoinRussia", "volcano in Russia", tweet)
 tweet = re.sub(r"ZippedNews", "Zipped News", tweet)
 tweet = re.sub(r"MicheleBachman", "Michele Bachman", tweet)
 tweet = re.sub(r"53inch", "53 inch", tweet)
 tweet = re.sub(r"KerrickTrial", "Kerrick Trial", tweet)
 tweet = re.sub(r"abstorm", "Alberta Storm", tweet)
 tweet = re.sub(r"Beyhive", "Beyonce hive", tweet)
 tweet = re.sub(r"RockyFire", "Rocky Fire", tweet)
 tweet = re.sub(r"Listen/Buy", "Listen / Buy", tweet)
 tweet = re.sub(r"ArtistsUnited", "Artists United", tweet)
 tweet = re.sub(r"ENGvAUS", "England vs Australia", tweet)
 tweet = re.sub(r"ScottWalker", "Scott Walker", tweet)
 return tweet
total['text'] = total['text'].apply(clean )
count_sgd = SGDClassifier()
count_sgd.fit(count_train ,y_train)
 count_sgd_pred = count_sgd.predict(count_test)
 count_sgd_score = accuracy_score(y_test,count_sgd_pred)
 print('SGD Count Score: ', count_sgd_score)
 count_sgd_cm = confusion_matrix(y_test, count_sgd_pred)
 count_sgd_cm<compute_train_metric>
tweets = [tweet for tweet in total['text']]
train = total[:len(train)]
test = total[len(train):]
count_lr = LogisticRegression()
count_lr.fit(count_train ,y_train)
 count_lr_pred = count_lr.predict(count_test)
 count_lr_score = accuracy_score(y_test,count_lr_pred)
 print('LogisticRegression Count Score: ', count_lr_score)
 count_lr_cm = confusion_matrix(y_test, count_lr_pred)
 count_lr_cm<feature_engineering>
def generate_ngrams(text, n_gram=1):
token = [token for token in text.lower().split(' ')if token != '' if token not in wordcloud.STOPWORDS]
ngrams = zip(*[token[i:] for i in range(n_gram)])
 return [' '.join(ngram)for ngram in ngrams]
disaster_unigrams = defaultdict(int)
 for word in total[train['target'] == 1]['text']:
for word in generate_ngrams(word, n_gram = 1):
disaster_unigrams[word] += 1
disaster_unigrams = pd.DataFrame(sorted(disaster_unigrams.items() , key=lambda x: x[1])[::-1])
 nondisaster_unigrams = defaultdict(int)
 for word in total[train['target'] == 0]['text']:
for word in generate_ngrams(word, n_gram = 1):
nondisaster_unigrams[word] += 1
nondisaster_unigrams = pd.DataFrame(sorted(nondisaster_unigrams.items() , key=lambda x: x[1])[::-1])
 disaster_bigrams = defaultdict(int)
 for word in total[train['target'] == 1]['text']:
for word in generate_ngrams(word, n_gram = 2):
disaster_bigrams[word] += 1
disaster_bigrams = pd.DataFrame(sorted(disaster_bigrams.items() , key=lambda x: x[1])[::-1])
 nondisaster_bigrams = defaultdict(int)
 for word in total[train['target'] == 0]['text']:
for word in generate_ngrams(word, n_gram = 2):
nondisaster_bigrams[word] += 1
nondisaster_bigrams = pd.DataFrame(sorted(nondisaster_bigrams.items() , key=lambda x: x[1])[::-1])
 disaster_trigrams = defaultdict(int)
 for word in total[train['target'] == 1]['text']:
for word in generate_ngrams(word, n_gram = 3):
disaster_trigrams[word] += 1
disaster_trigrams = pd.DataFrame(sorted(disaster_trigrams.items() , key=lambda x: x[1])[::-1])
 nondisaster_trigrams = defaultdict(int)
 for word in total[train['target'] == 0]['text']:
for word in generate_ngrams(word, n_gram = 3):
nondisaster_trigrams[word] += 1
nondisaster_trigrams = pd.DataFrame(sorted(nondisaster_trigrams.items() , key=lambda x: x[1])[::-1])
 disaster_4grams = defaultdict(int)
 for word in total[train['target'] == 1]['text']:
for word in generate_ngrams(word, n_gram = 4):
disaster_4grams[word] += 1
disaster_4grams = pd.DataFrame(sorted(disaster_4grams.items() , key=lambda x: x[1])[::-1])
 nondisaster_4grams = defaultdict(int)
 for word in total[train['target'] == 0]['text']:
for word in generate_ngrams(word, n_gram = 4):
nondisaster_4grams[word] += 1
nondisaster_4grams = pd.DataFrame(sorted(nondisaster_4grams.items() , key=lambda x: x[1])[::-1] )
tfidf_vectorizer = TfidfVectorizer(stop_words='english')
 tfidf_train = tfidf_vectorizer.fit_transform(X_train)
 tfidf_test = tfidf_vectorizer.transform(X_test)
 tfidf_train_sub = tfidf_vectorizer.transform(X)
 tfidf_sub = tfidf_vectorizer.transform(test_x )<compute_train_metric>
to_exclude = '*+-/() %
[\\]{|}^_`~\t'
to_tokenize = '!"
tokenizer = Tokenizer(filters = to_exclude)
 text = 'Why are you so f%
text = re.sub(r'(['+to_tokenize+'])', r' \1 ', text)
 tokenizer.fit_on_texts([text])
 print(tokenizer.word_index )
tfidf_nb = MultinomialNB()
tfidf_nb.fit(tfidf_train, y_train)
 tfidf_nb_pred = tfidf_nb.predict(tfidf_test)
 tfidf_nb_score = accuracy_score(y_test,tfidf_nb_pred)
 print('MultinomialNaiveBayes Tfidf Score: ', tfidf_nb_score)
 tfidf_nb_cm = confusion_matrix(y_test, tfidf_nb_pred)
 tfidf_nb_cm<compute_train_metric>
tfidf_svc = LinearSVC()
tfidf_svc.fit(tfidf_train, y_train)
 tfidf_svc_pred = tfidf_svc.predict(tfidf_test)
 tfidf_svc_score = accuracy_score(y_test,tfidf_svc_pred)
 print("LinearSVC Score: %0.3f" % tfidf_svc_score)
 svc_cm = confusion_matrix(y_test, tfidf_svc_pred)
 svc_cm<compute_train_metric>
tokenizer = Tokenizer()
tokenizer.fit_on_texts(tweets)
 sequences = tokenizer.texts_to_sequences(tweets)
 word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
data = pad_sequences(sequences)
 labels = train['target']
print('Shape of data tensor:', data.shape)
 print('Shape of label tensor:', labels.shape)
 nlp_train = data[:len(train)]
labels = labels
nlp_test = data[len(train):]
MAX_SEQUENCE_LENGTH = data.shape[1]
tfidf_svc0 = SVC()
tfidf_svc0.fit(tfidf_train, y_train)
 tfidf_svc_pred0 = tfidf_svc.predict(tfidf_test)
 tfidf_svc_score0 = accuracy_score(y_test,tfidf_svc_pred0)
 print("SVC Score: %0.3f" % tfidf_svc_score0)
 svc_cm0 = confusion_matrix(y_test, tfidf_svc_pred0)
 classification_report(y_test, tfidf_svc_pred0)
 svc_cm0<compute_train_metric>
embeddings_index = {}
with open('.. /input/glove-global-vectors-for-word-representation/glove.6B.200d.txt','r')as f:
for line in tqdm(f):
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
print('Found %s word vectors in the GloVe library' % len(embeddings_index))
tfidf_nusvc = NuSVC()
tfidf_nusvc.fit(tfidf_train, y_train)
 tfidf_nusvc_pred = tfidf_nusvc.predict(tfidf_test)
 tfidf_nusvc_score = accuracy_score(y_test,tfidf_nusvc_pred)
 print("NuSVC Score: %0.3f" % tfidf_nusvc_score)
 nusvc_cm = confusion_matrix(y_test, tfidf_nusvc_pred)
 classification_report(y_test, tfidf_nusvc_pred)
 nusvc_cm<compute_train_metric>
EMBEDDING_DIM = 200
tfidf_bnb = BernoulliNB()
tfidf_bnb.fit(tfidf_train, y_train)
 tfidf_bnb_pred = tfidf_bnb.predict(tfidf_test)
 tfidf_bnb_score = accuracy_score(y_test,tfidf_bnb_pred)
 print('BernoulliNaiveBayes Tfidf Score: %0.3f' % tfidf_bnb_score)
 tfidf_bnb_cm = confusion_matrix(y_test, tfidf_bnb_pred)
 tfidf_bnb_cm<compute_train_metric>
embedding_matrix = np.zeros(( len(word_index)+ 1, EMBEDDING_DIM))
for word, i in tqdm(word_index.items()):
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
print("Our embedded matrix is of dimension", embedding_matrix.shape )
tfidf_sgd = SGDClassifier()
tfidf_sgd.fit(tfidf_train, y_train)
 tfidf_sgd_pred = tfidf_sgd.predict(tfidf_test)
 tfidf_sgd_score = accuracy_score(y_test,tfidf_sgd_pred)
 print("SGD Score: %0.3f" % tfidf_sgd_score)
 sgd_cm = confusion_matrix(y_test, tfidf_sgd_pred)
 sgd_cm<compute_train_metric>
embedding = Embedding(len(word_index)+ 1, EMBEDDING_DIM, weights = [embedding_matrix],
input_length = MAX_SEQUENCE_LENGTH, trainable = False)
tfidf_lr = LogisticRegression()
tfidf_lr.fit(tfidf_train, y_train)
 tfidf_lr_pred = tfidf_lr.predict(tfidf_test)
 tfidf_lr_score = accuracy_score(y_test,tfidf_lr_pred)
 print("LogisticRegression Score: %0.3f" % tfidf_lr_score)
 lr_cm = confusion_matrix(y_test, tfidf_lr_pred)
 lr_cm<load_from_csv>
def scale(df, scaler):
return scaler.fit_transform(df.iloc[:, 2:])
 meta_train = scale(train, StandardScaler())
 meta_test = scale(test, StandardScaler() )
sample_sub=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
<predict_on_test>
def create_lstm(spatial_dropout, dropout, recurrent_dropout, learning_rate, bidirectional = False):
activation = LeakyReLU(alpha = 0.01)
 nlp_input = Input(shape =(MAX_SEQUENCE_LENGTH,), name = 'nlp_input')
 meta_input_train = Input(shape =(7,), name = 'meta_train')
 emb = embedding(nlp_input)
 emb = SpatialDropout1D(dropout )(emb)
 if bidirectional:
nlp_out =(Bidirectional(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal')) )(emb)
 else:
nlp_out =(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal'))(emb)
 x = Concatenate()([nlp_out, meta_input_train])
 x = Dropout(dropout )(x)
 preds = Dense(1, activation='sigmoid', kernel_regularizer = regularizers.l2(1e-4))(x)
 model = Model(inputs=[nlp_input , meta_input_train], outputs = preds)
 optimizer = Adam(learning_rate = learning_rate)
 model.compile(loss = 'binary_crossentropy', optimizer = optimizer, metrics = ['accuracy'])
 return model
count_nusvc.fit(count_train_sub ,y)
 count_nusvc_sub = count_nusvc.predict(count_sub)
<create_dataframe>
lstm = create_lstm(spatial_dropout =.2, dropout =.2, recurrent_dropout =.2,
learning_rate = 3e-4, bidirectional = True)
 lstm.summary()
sub=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':count_nusvc_sub} )<save_to_csv>
history1 = lstm.fit([nlp_train, meta_train], labels, validation_split =.2,
epochs = 5, batch_size = 21, verbose = 1 )
sub.to_csv('submission.csv',index=False )<import_modules>
callback = EarlyStopping(monitor = 'val_loss', patience = 4)
import nltk
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer
from sklearn.multioutput import MultiOutputClassifier
from sklearn.metrics import classification_report, accuracy_score, precision_score, recall_score, plot_confusion_matrix
from sklearn.svm import LinearSVC, SVC, NuSVC
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.naive_bayes import MultinomialNB, BernoulliNB
import random<define_variables>
def create_lstm_2(spatial_dropout, dropout, recurrent_dropout, learning_rate, bidirectional = False):
activation = LeakyReLU(alpha = 0.01)
 nlp_input = Input(shape =(MAX_SEQUENCE_LENGTH,), name = 'nlp_input')
 meta_input_train = Input(shape =(7,), name = 'meta_train')
 emb = embedding(nlp_input)
 emb = SpatialDropout1D(dropout )(emb)
 if bidirectional:
nlp_out =(Bidirectional(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal')) )(emb)
 else:
nlp_out =(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal'))(emb)
 x = Concatenate()([nlp_out, meta_input_train])
 x = Dropout(dropout )(x)
 x =(Dense(100, activation = activation, kernel_regularizer = regularizers.l2(1e-4),
kernel_initializer = 'he_normal'))(x)
 x = Dropout(dropout )(x)
 preds = Dense(1, activation='sigmoid', kernel_regularizer = regularizers.l2(1e-4))(x)
 model = Model(inputs=[nlp_input , meta_input_train], outputs = preds)
 optimizer = Adam(learning_rate = learning_rate)
 model.compile(loss = 'binary_crossentropy', optimizer = optimizer, metrics = ['accuracy'])
 return model
rand_state = random.seed(12 )<load_from_csv>
lstm_2 = create_lstm_2(spatial_dropout =.4, dropout =.4, recurrent_dropout =.4,
learning_rate = 3e-4, bidirectional = True)
 lstm_2.summary()
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )<prepare_x_and_y>
history2 = lstm_2.fit([nlp_train, meta_train], labels, validation_split =.2,
epochs = 30, batch_size = 21, verbose = 1 )
X = train['text']<prepare_x_and_y>
submission_lstm = pd.DataFrame()
submission_lstm['id'] = test_id
submission_lstm['prob'] = lstm_2.predict([nlp_test, meta_test])
 submission_lstm['target'] = submission_lstm['prob'].apply(lambda x: 0 if x <.5 else 1)
 submission_lstm.head(10 )
y = train['target']<prepare_x_and_y>
def create_dual_lstm(spatial_dropout, dropout, recurrent_dropout, learning_rate, bidirectional = False):
activation = LeakyReLU(alpha = 0.01)
 nlp_input = Input(shape =(MAX_SEQUENCE_LENGTH,), name = 'nlp_input')
 meta_input_train = Input(shape =(7,), name = 'meta_train')
 emb = embedding(nlp_input)
 emb = SpatialDropout1D(dropout )(emb)
 if bidirectional:
nlp_out =(Bidirectional(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal', return_sequences = True)) )(emb)
 nlp_out = SpatialDropout1D(dropout )(nlp_out)
 nlp_out =(Bidirectional(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal')) )(emb)
 else:
nlp_out =(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal', return_sequences = True))(emb)
 nlp_out = SpatialDropout1D(dropout )(nlp_out)
 nlp_out =(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal'))(emb)
 x = Concatenate()([nlp_out, meta_input_train])
 x = Dropout(dropout )(x)
 preds = Dense(1, activation='sigmoid', kernel_regularizer = regularizers.l2(1e-4))(x)
 model = Model(inputs=[nlp_input , meta_input_train], outputs = preds)
 optimizer = Adam(learning_rate = learning_rate)
 model.compile(loss = 'binary_crossentropy', optimizer = optimizer, metrics = ['accuracy'])
 return model
test_x = test['text']<split>
history3 = dual_lstm.fit([nlp_train, meta_train], labels, validation_split =.2,
epochs = 25, batch_size = 21, verbose = 1 )
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state = rand_state, shuffle = True )<feature_engineering>
submission_lstm2 = pd.DataFrame()
submission_lstm2['id'] = test_id
submission_lstm2['prob'] = dual_lstm.predict([nlp_test, meta_test])
 submission_lstm2['target'] = submission_lstm2['prob'].apply(lambda x: 0 if x <.5 else 1)
 submission_lstm2.head(10 )
count_vectorizer = CountVectorizer(stop_words='english')
 count_train = count_vectorizer.fit_transform(X_train)
 count_test = count_vectorizer.transform(X_test)
 count_train_sub = count_vectorizer.transform(X)
 count_sub = count_vectorizer.transform(test_x)
<compute_train_metric>
BATCH_SIZE = 32
EPOCHS = 2
USE_META = True
ADD_DENSE = False
DENSE_DIM = 64
ADD_DROPOUT = False
DROPOUT =.2
count_nb = MultinomialNB()
count_nb.fit(count_train ,y_train)
 count_nb_pred = count_nb.predict(count_test)
 count_nb_score = accuracy_score(y_test,count_nb_pred)
 print('MultinomialNaiveBayes Count Score: ', count_nb_score)
 count_nb_cm = confusion_matrix(y_test, count_nb_pred)
 count_nb_cm<compute_train_metric>
!pip install --quiet transformers
count_bnb = BernoulliNB()
count_bnb.fit(count_train ,y_train)
 count_bnb_pred = count_bnb.predict(count_test)
 count_bnb_score = accuracy_score(y_test,count_bnb_pred)
 print('BernoulliNaiveBayes Count Score: ', count_bnb_score)
 count_bnb_cm = confusion_matrix(y_test, count_bnb_pred)
 count_bnb_cm<compute_train_metric>
TOKENIZER = AutoTokenizer.from_pretrained("bert-large-uncased")
 enc = TOKENIZER.encode("Encode me!")
 dec = TOKENIZER.decode(enc)
 print("Encode: " + str(enc))
print("Decode: " + str(dec))
count_lsvc = LinearSVC()
count_lsvc.fit(count_train ,y_train)
 count_lsvc_pred = count_lsvc.predict(count_test)
 count_lsvc_score = accuracy_score(y_test,count_lsvc_pred)
 print('LinearSVC Count Score: ', count_lsvc_score)
 count_lsvc_cm = confusion_matrix(y_test, count_lsvc_pred)
 count_lsvc_cm<compute_train_metric>
def bert_encode(data,maximum_len):
input_ids = []
attention_masks = []
for i in range(len(data.text)) :
encoded = TOKENIZER.encode_plus(data.text[i],
add_special_tokens=True,
max_length=maximum_len,
pad_to_max_length=True,
return_attention_mask=True)
 input_ids.append(encoded['input_ids'])
 attention_masks.append(encoded['attention_mask'])
 return np.array(input_ids),np.array(attention_masks )
count_svc = SVC()
count_svc.fit(count_train ,y_train)
 count_svc_pred = count_svc.predict(count_test)
 count_svc_score = accuracy_score(y_test,count_svc_pred)
 print('SVC Count Score: ', count_svc_score)
 count_svc_cm = confusion_matrix(y_test, count_svc_pred)
 count_svc_cm<compute_train_metric>
def build_model(model_layer, learning_rate, use_meta = USE_META, add_dense = ADD_DENSE,
dense_dim = DENSE_DIM, add_dropout = ADD_DROPOUT, dropout = DROPOUT):
input_ids = tf.keras.Input(shape=(60,),dtype='int32')
 attention_masks = tf.keras.Input(shape=(60,),dtype='int32')
 meta_input = tf.keras.Input(shape =(meta_train.shape[1],))
transformer_layer = model_layer([input_ids,attention_masks])
 output = transformer_layer[1]
if use_meta:
output = tf.keras.layers.Concatenate()([output, meta_input])
 if add_dense:
print("Training with additional dense layer...")
 output = tf.keras.layers.Dense(dense_dim,activation='relu' )(output)
 if add_dropout:
print("Training with dropout...")
 output = tf.keras.layers.Dropout(dropout )(output)
 output = tf.keras.layers.Dense(1,activation='sigmoid' )(output)
 if use_meta:
print("Training with meta-data...")
 model = tf.keras.models.Model(inputs = [input_ids,attention_masks, meta_input],outputs = output)
 else:
print("Training without meta-data...")
 model = tf.keras.models.Model(inputs = [input_ids,attention_masks],outputs = output)
 model.compile(tf.keras.optimizers.Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy'])
 return model
count_nusvc = NuSVC(0.4)
 count_nusvc.fit(count_train ,y_train)
 count_nusvc_pred = count_nusvc.predict(count_test)
 count_nusvc_score = accuracy_score(y_test,count_nusvc_pred)
 print('NuSVC Count Score: ', count_nusvc_score)
 count_nusvc_cm = confusion_matrix(y_test, count_nusvc_pred)
 count_nusvc_cm<train_on_grid>
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv' )
<compute_test_metric>
bert_large = TFAutoModel.from_pretrained('bert-large-uncased')
 TOKENIZER = AutoTokenizer.from_pretrained("bert-large-uncased")
 train_input_ids,train_attention_masks = bert_encode(train,60)
 test_input_ids,test_attention_masks = bert_encode(test,60)
 print('Train length:', len(train_input_ids))
print('Test length:', len(test_input_ids))
BERT_large = build_model(bert_large, learning_rate = 1e-5)
 BERT_large.summary()
<compute_train_metric>
history_bert = BERT_large.fit([train_input_ids,train_attention_masks, meta_train], train.target,
validation_split =.2, epochs = EPOCHS, callbacks = [checkpoint], batch_size = BATCH_SIZE )
count_sgd = SGDClassifier()
count_sgd.fit(count_train ,y_train)
 count_sgd_pred = count_sgd.predict(count_test)
 count_sgd_score = accuracy_score(y_test,count_sgd_pred)
 print('SGD Count Score: ', count_sgd_score)
 count_sgd_cm = confusion_matrix(y_test, count_sgd_pred)
 count_sgd_cm<compute_train_metric>
BERT_large.load_weights('large_model.h5')
 preds_bert = BERT_large.predict([test_input_ids,test_attention_masks,meta_test] )
count_lr = LogisticRegression()
count_lr.fit(count_train ,y_train)
 count_lr_pred = count_lr.predict(count_test)
 count_lr_score = accuracy_score(y_test,count_lr_pred)
 print('LogisticRegression Count Score: ', count_lr_score)
 count_lr_cm = confusion_matrix(y_test, count_lr_pred)
 count_lr_cm<feature_engineering>
submission_bert = pd.DataFrame()
submission_bert['id'] = test_id
submission_bert['prob'] = preds_bert
submission_bert['target'] = np.round(submission_bert['prob'] ).astype(int)
 submission_bert.head(10 )
tfidf_vectorizer = TfidfVectorizer(stop_words='english')
 tfidf_train = tfidf_vectorizer.fit_transform(X_train)
 tfidf_test = tfidf_vectorizer.transform(X_test)
 tfidf_train_sub = tfidf_vectorizer.transform(X)
 tfidf_sub = tfidf_vectorizer.transform(test_x )<compute_train_metric>
submission_bert = submission_bert[['id', 'target']]
submission_bert.to_csv('submission_bert.csv', index = False)
 print('Blended submission has been saved to disk' )
tfidf_nb = MultinomialNB()
tfidf_nb.fit(tfidf_train, y_train)
 tfidf_nb_pred = tfidf_nb.predict(tfidf_test)
 tfidf_nb_score = accuracy_score(y_test,tfidf_nb_pred)
 print('MultinomialNaiveBayes Tfidf Score: ', tfidf_nb_score)
 tfidf_nb_cm = confusion_matrix(y_test, tfidf_nb_pred)
 tfidf_nb_cm<compute_train_metric>
plt.style.use('ggplot')
 warnings.filterwarnings('ignore')
tfidf_svc = LinearSVC()
tfidf_svc.fit(tfidf_train, y_train)
 tfidf_svc_pred = tfidf_svc.predict(tfidf_test)
 tfidf_svc_score = accuracy_score(y_test,tfidf_svc_pred)
 print("LinearSVC Score: %0.3f" % tfidf_svc_score)
 svc_cm = confusion_matrix(y_test, tfidf_svc_pred)
 svc_cm<compute_train_metric>
def seed_everything(seed):
os.environ['PYTHONHASHSEED']=str(seed)
 tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 seed_everything(34 )
tfidf_svc0 = SVC()
tfidf_svc0.fit(tfidf_train, y_train)
 tfidf_svc_pred0 = tfidf_svc.predict(tfidf_test)
 tfidf_svc_score0 = accuracy_score(y_test,tfidf_svc_pred0)
 print("SVC Score: %0.3f" % tfidf_svc_score0)
 svc_cm0 = confusion_matrix(y_test, tfidf_svc_pred0)
 classification_report(y_test, tfidf_svc_pred0)
 svc_cm0<compute_train_metric>
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 train.head()
tfidf_nusvc = NuSVC()
tfidf_nusvc.fit(tfidf_train, y_train)
 tfidf_nusvc_pred = tfidf_nusvc.predict(tfidf_test)
 tfidf_nusvc_score = accuracy_score(y_test,tfidf_nusvc_pred)
 print("NuSVC Score: %0.3f" % tfidf_nusvc_score)
 nusvc_cm = confusion_matrix(y_test, tfidf_nusvc_pred)
 classification_report(y_test, tfidf_nusvc_pred)
 nusvc_cm<compute_train_metric>
test_id = test['id']
columns = {'id', 'location'}
train = train.drop(columns = columns)
 test = test.drop(columns = columns)
 train['keyword'] = train['keyword'].fillna('unknown')
 test['keyword'] = test['keyword'].fillna('unknown')
 train['text'] = train['text'] + ' ' + train['keyword']
test['text'] = test['text'] + ' ' + test['keyword']
columns = {'keyword'}
train = train.drop(columns = columns)
 test = test.drop(columns = columns)
 total = train.append(test )
tfidf_bnb = BernoulliNB()
tfidf_bnb.fit(tfidf_train, y_train)
 tfidf_bnb_pred = tfidf_bnb.predict(tfidf_test)
 tfidf_bnb_score = accuracy_score(y_test,tfidf_bnb_pred)
 print('BernoulliNaiveBayes Tfidf Score: %0.3f' % tfidf_bnb_score)
 tfidf_bnb_cm = confusion_matrix(y_test, tfidf_bnb_pred)
 tfidf_bnb_cm<compute_train_metric>
total['unique word count'] = total['text'].apply(lambda x: len(set(x.split())))
 total['stopword count'] = total['text'].apply(lambda x: len([i for i in x.lower().split() if i in wordcloud.STOPWORDS]))
total['stopword ratio'] = total['stopword count'] / total['word count']
total['punctuation count'] = total['text'].apply(lambda x: len([i for i in str(x)if i in string.punctuation]))
train = total[:len(train)]
disaster = train['target'] == 1
fig, axes = plt.subplots(4, figsize=(20, 30))
graph1 = sns.kdeplot(train.loc[~disaster]['unique word count'], shade = True, label = 'Not Disaster', ax=axes[0])
 graph1 = sns.kdeplot(train.loc[disaster]['unique word count'], shade = True, label = 'Disaster', ax=axes[0])
 graph1.set_title('Distribution of Unique Word Count')
 graph2 = sns.kdeplot(train.loc[~disaster]['stopword count'], shade = True, label = 'Not Disaster', ax=axes[1])
 graph2 = sns.kdeplot(train.loc[disaster]['stopword count'], shade = True, label = 'Disaster', ax=axes[1])
 graph2.set_title('Distribution of Stopword Word Count')
 graph3 = sns.kdeplot(train.loc[~disaster]['punctuation count'], shade = True, label = 'Not Disaster', ax=axes[2], bw = 1)
 graph3 = sns.kdeplot(train.loc[disaster]['punctuation count'], shade = True, label = 'Disaster', ax=axes[2], bw = 1)
 graph3.set_title('Distribution of Punctuation Count')
 graph4 = sns.kdeplot(train.loc[~disaster]['stopword ratio'], shade = True, label = 'Not Disaster', ax=axes[3], bw =.05)
 graph4 = sns.kdeplot(train.loc[disaster]['stopword ratio'], shade = True, label = 'Disaster', ax=axes[3], bw =.05)
 graph4.set_title('Distribution of Stopword Ratio')
 fig.tight_layout()
plt.show()
tfidf_sgd = SGDClassifier()
tfidf_sgd.fit(tfidf_train, y_train)
 tfidf_sgd_pred = tfidf_sgd.predict(tfidf_test)
 tfidf_sgd_score = accuracy_score(y_test,tfidf_sgd_pred)
 print("SGD Score: %0.3f" % tfidf_sgd_score)
 sgd_cm = confusion_matrix(y_test, tfidf_sgd_pred)
 sgd_cm<compute_train_metric>
def remove_punctuation(x):
return x.translate(str.maketrans('', '', string.punctuation))
def remove_stopwords(x):
return ' '.join([i for i in x.split() if i not in wordcloud.STOPWORDS])
 def remove_less_than(x):
return ' '.join([i for i in x.split() if len(i)> 3])
 def remove_non_alphabet(x):
return ' '.join([i for i in x.split() if i.isalpha() ])
 def strip_all_entities(x):
return ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)"," ",x ).split() )
tfidf_lr = LogisticRegression()
tfidf_lr.fit(tfidf_train, y_train)
 tfidf_lr_pred = tfidf_lr.predict(tfidf_test)
 tfidf_lr_score = accuracy_score(y_test,tfidf_lr_pred)
 print("LogisticRegression Score: %0.3f" % tfidf_lr_score)
 lr_cm = confusion_matrix(y_test, tfidf_lr_pred)
 lr_cm<load_from_csv>
strip_all_entities('@shawn Titanic
Times: Telegraph.co.ukTitanic tragedy could have been preve...http://bet.ly/tuN2wx' )
sample_sub=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
<predict_on_test>
!pip install autocorrect
def spell_check(x):
spell = Speller(lang='en')
 return " ".join([spell(i)for i in x.split() ])
 mispelled = 'Pleaze spelcheck this sentince'
spell_check(mispelled )
count_nusvc.fit(count_train_sub ,y)
 count_nusvc_sub = count_nusvc.predict(count_sub)
<create_dataframe>
PROCESS_TWEETS = False
if PROCESS_TWEETS:
total['text'] = total['text'].apply(lambda x: x.lower())
 total['text'] = total['text'].apply(lambda x: re.sub(r'https?://\S+|www\.\S+', '', x, flags = re.MULTILINE))
total['text'] = total['text'].apply(remove_punctuation)
 total['text'] = total['text'].apply(remove_stopwords)
 total['text'] = total['text'].apply(remove_less_than)
 total['text'] = total['text'].apply(remove_non_alphabet)
 total['text'] = total['text'].apply(spell_check )
sub=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':count_nusvc_sub} )<save_to_csv>
contractions = {
"ain't": "am not / are not / is not / has not / have not",
"aren't": "are not / am not",
"can't": "cannot",
"can't've": "cannot have",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"couldn't've": "could not have",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hadn't've": "had not have",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he had / he would",
"he'd've": "he would have",
"he'll": "he shall / he will",
"he'll've": "he shall have / he will have",
"he's": "he has / he is",
"how'd": "how did",
"how'd'y": "how do you",
"how'll": "how will",
"how's": "how has / how is / how does",
"I'd": "I had / I would",
"I'd've": "I would have",
"I'll": "I shall / I will",
"I'll've": "I shall have / I will have",
"I'm": "I am",
"I've": "I have",
"isn't": "is not",
"it'd": "it had / it would",
"it'd've": "it would have",
"it'll": "it shall / it will",
"it'll've": "it shall have / it will have",
"it's": "it has / it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"mightn't've": "might not have",
"must've": "must have",
"mustn't": "must not",
"mustn't've": "must not have",
"needn't": "need not",
"needn't've": "need not have",
"o'clock": "of the clock",
"oughtn't": "ought not",
"oughtn't've": "ought not have",
"shan't": "shall not",
"sha'n't": "shall not",
"shan't've": "shall not have",
"she'd": "she had / she would",
"she'd've": "she would have",
"she'll": "she shall / she will",
"she'll've": "she shall have / she will have",
"she's": "she has / she is",
"should've": "should have",
"shouldn't": "should not",
"shouldn't've": "should not have",
"so've": "so have",
"so's": "so as / so is",
"that'd": "that would / that had",
"that'd've": "that would have",
"that's": "that has / that is",
"there'd": "there had / there would",
"there'd've": "there would have",
"there's": "there has / there is",
"they'd": "they had / they would",
"they'd've": "they would have",
"they'll": "they shall / they will",
"they'll've": "they shall have / they will have",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd": "we had / we would",
"we'd've": "we would have",
"we'll": "we will",
"we'll've": "we will have",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what shall / what will",
"what'll've": "what shall have / what will have",
"what're": "what are",
"what's": "what has / what is",
"what've": "what have",
"when's": "when has / when is",
"when've": "when have",
"where'd": "where did",
"where's": "where has / where is",
"where've": "where have",
"who'll": "who shall / who will",
"who'll've": "who shall have / who will have",
"who's": "who has / who is",
"who've": "who have",
"why's": "why has / why is",
"why've": "why have",
"will've": "will have",
"won't": "will not",
"won't've": "will not have",
"would've": "would have",
"wouldn't": "would not",
"wouldn't've": "would not have",
"y'all": "you all",
"y'all'd": "you all would",
"y'all'd've": "you all would have",
"y'all're": "you all are",
"y'all've": "you all have",
"you'd": "you had / you would",
"you'd've": "you would have",
"you'll": "you shall / you will",
"you'll've": "you shall have / you will have",
"you're": "you are",
"you've": "you have"
}
contractions_re = re.compile('(%s)' % '|'.join(contractions.keys()))
def expand_contractions(s, contractions = contractions):
def replace(match):
return contractions[match.group(0)]
return contractions_re.sub(replace, s)
 expand_contractions("can't stop won't stop" )
sub.to_csv('submission.csv',index=False )<import_modules>
total['text'] = total['text'].apply(expand_contractions )
from sklearn.linear_model import LogisticRegression<load_from_csv>
def clean(tweet):
tweet = re.sub(r"tnwx", "Tennessee Weather", tweet)
 tweet = re.sub(r"azwx", "Arizona Weather", tweet)
 tweet = re.sub(r"alwx", "Alabama Weather", tweet)
 tweet = re.sub(r"wordpressdotcom", "wordpress", tweet)
 tweet = re.sub(r"gawx", "Georgia Weather", tweet)
 tweet = re.sub(r"scwx", "South Carolina Weather", tweet)
 tweet = re.sub(r"cawx", "California Weather", tweet)
 tweet = re.sub(r"usNWSgov", "United States National Weather Service", tweet)
 tweet = re.sub(r"MH370", "Malaysia Airlines Flight 370", tweet)
 tweet = re.sub(r"okwx", "Oklahoma City Weather", tweet)
 tweet = re.sub(r"arwx", "Arkansas Weather", tweet)
 tweet = re.sub(r"lmao", "laughing my ass off", tweet)
 tweet = re.sub(r"amirite", "am I right", tweet)
 tweet = re.sub(r"w/e", "whatever", tweet)
 tweet = re.sub(r"w/", "with", tweet)
 tweet = re.sub(r"USAgov", "USA government", tweet)
 tweet = re.sub(r"recentlu", "recently", tweet)
 tweet = re.sub(r"Ph0tos", "Photos", tweet)
 tweet = re.sub(r"exp0sed", "exposed", tweet)
 tweet = re.sub(r"<3", "love", tweet)
 tweet = re.sub(r"amageddon", "armageddon", tweet)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"WindStorm", "Wind Storm", tweet)
 tweet = re.sub(r"16yr", "16 year", tweet)
 tweet = re.sub(r"TRAUMATISED", "traumatized", tweet)
 tweet = re.sub(r"IranDeal", "Iran Deal", tweet)
 tweet = re.sub(r"ArianaGrande", "Ariana Grande", tweet)
 tweet = re.sub(r"camilacabello97", "camila cabello", tweet)
 tweet = re.sub(r"RondaRousey", "Ronda Rousey", tweet)
 tweet = re.sub(r"MTVHottest", "MTV Hottest", tweet)
 tweet = re.sub(r"TrapMusic", "Trap Music", tweet)
 tweet = re.sub(r"ProphetMuhammad", "Prophet Muhammad", tweet)
 tweet = re.sub(r"PantherAttack", "Panther Attack", tweet)
 tweet = re.sub(r"StrategicPatience", "Strategic Patience", tweet)
 tweet = re.sub(r"socialnews", "social news", tweet)
 tweet = re.sub(r"IDPs:", "Internally Displaced People :", tweet)
 tweet = re.sub(r"ArtistsUnited", "Artists United", tweet)
 tweet = re.sub(r"ClaytonBryant", "Clayton Bryant", tweet)
 tweet = re.sub(r"jimmyfallon", "jimmy fallon", tweet)
 tweet = re.sub(r"justinbieber", "justin bieber", tweet)
 tweet = re.sub(r"Time2015", "Time 2015", tweet)
 tweet = re.sub(r"djicemoon", "dj icemoon", tweet)
 tweet = re.sub(r"LivingSafely", "Living Safely", tweet)
 tweet = re.sub(r"FIFA16", "Fifa 2016", tweet)
 tweet = re.sub(r"thisiswhywecanthavenicethings", "this is why we cannot have nice things", tweet)
 tweet = re.sub(r"bbcnews", "bbc news", tweet)
 tweet = re.sub(r"UndergroundRailraod", "Underground Railraod", tweet)
 tweet = re.sub(r"c4news", "c4 news", tweet)
 tweet = re.sub(r"MUDSLIDE", "mudslide", tweet)
 tweet = re.sub(r"NoSurrender", "No Surrender", tweet)
 tweet = re.sub(r"NotExplained", "Not Explained", tweet)
 tweet = re.sub(r"greatbritishbakeoff", "great british bake off", tweet)
 tweet = re.sub(r"LondonFire", "London Fire", tweet)
 tweet = re.sub(r"KOTAWeather", "KOTA Weather", tweet)
 tweet = re.sub(r"LuchaUnderground", "Lucha Underground", tweet)
 tweet = re.sub(r"KOIN6News", "KOIN 6 News", tweet)
 tweet = re.sub(r"LiveOnK2", "Live On K2", tweet)
 tweet = re.sub(r"9NewsGoldCoast", "9 News Gold Coast", tweet)
 tweet = re.sub(r"nikeplus", "nike plus", tweet)
 tweet = re.sub(r"david_cameron", "David Cameron", tweet)
 tweet = re.sub(r"peterjukes", "Peter Jukes", tweet)
 tweet = re.sub(r"MikeParrActor", "Michael Parr", tweet)
 tweet = re.sub(r"4PlayThursdays", "Foreplay Thursdays", tweet)
 tweet = re.sub(r"TGF2015", "Tontitown Grape Festival", tweet)
 tweet = re.sub(r"realmandyrain", "Mandy Rain", tweet)
 tweet = re.sub(r"GraysonDolan", "Grayson Dolan", tweet)
 tweet = re.sub(r"ApolloBrown", "Apollo Brown", tweet)
 tweet = re.sub(r"saddlebrooke", "Saddlebrooke", tweet)
 tweet = re.sub(r"TontitownGrape", "Tontitown Grape", tweet)
 tweet = re.sub(r"AbbsWinston", "Abbs Winston", tweet)
 tweet = re.sub(r"ShaunKing", "Shaun King", tweet)
 tweet = re.sub(r"MeekMill", "Meek Mill", tweet)
 tweet = re.sub(r"TornadoGiveaway", "Tornado Giveaway", tweet)
 tweet = re.sub(r"GRupdates", "GR updates", tweet)
 tweet = re.sub(r"SouthDowns", "South Downs", tweet)
 tweet = re.sub(r"braininjury", "brain injury", tweet)
 tweet = re.sub(r"auspol", "Australian politics", tweet)
 tweet = re.sub(r"PlannedParenthood", "Planned Parenthood", tweet)
 tweet = re.sub(r"calgaryweather", "Calgary Weather", tweet)
 tweet = re.sub(r"weallheartonedirection", "we all heart one direction", tweet)
 tweet = re.sub(r"edsheeran", "Ed Sheeran", tweet)
 tweet = re.sub(r"TrueHeroes", "True Heroes", tweet)
 tweet = re.sub(r"ComplexMag", "Complex Magazine", tweet)
 tweet = re.sub(r"TheAdvocateMag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"CityofCalgary", "City of Calgary", tweet)
 tweet = re.sub(r"EbolaOutbreak", "Ebola Outbreak", tweet)
 tweet = re.sub(r"SummerFate", "Summer Fate", tweet)
 tweet = re.sub(r"RAmag", "Royal Academy Magazine", tweet)
 tweet = re.sub(r"offers2go", "offers to go", tweet)
 tweet = re.sub(r"ModiMinistry", "Modi Ministry", tweet)
 tweet = re.sub(r"TAXIWAYS", "taxi ways", tweet)
 tweet = re.sub(r"Calum5SOS", "Calum Hood", tweet)
 tweet = re.sub(r"JamesMelville", "James Melville", tweet)
 tweet = re.sub(r"JamaicaObserver", "Jamaica Observer", tweet)
 tweet = re.sub(r"TweetLikeItsSeptember11th2001", "Tweet like it is september 11th 2001", tweet)
 tweet = re.sub(r"cbplawyers", "cbp lawyers", tweet)
 tweet = re.sub(r"fewmoretweets", "few more tweets", tweet)
 tweet = re.sub(r"BlackLivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"NASAHurricane", "NASA Hurricane", tweet)
 tweet = re.sub(r"onlinecommunities", "online communities", tweet)
 tweet = re.sub(r"humanconsumption", "human consumption", tweet)
 tweet = re.sub(r"Typhoon-Devastated", "Typhoon Devastated", tweet)
 tweet = re.sub(r"Meat-Loving", "Meat Loving", tweet)
 tweet = re.sub(r"facialabuse", "facial abuse", tweet)
 tweet = re.sub(r"LakeCounty", "Lake County", tweet)
 tweet = re.sub(r"BeingAuthor", "Being Author", tweet)
 tweet = re.sub(r"withheavenly", "with heavenly", tweet)
 tweet = re.sub(r"thankU", "thank you", tweet)
 tweet = re.sub(r"iTunesMusic", "iTunes Music", tweet)
 tweet = re.sub(r"OffensiveContent", "Offensive Content", tweet)
 tweet = re.sub(r"WorstSummerJob", "Worst Summer Job", tweet)
 tweet = re.sub(r"HarryBeCareful", "Harry Be Careful", tweet)
 tweet = re.sub(r"NASASolarSystem", "NASA Solar System", tweet)
 tweet = re.sub(r"animalrescue", "animal rescue", tweet)
 tweet = re.sub(r"KurtSchlichter", "Kurt Schlichter", tweet)
 tweet = re.sub(r"Throwingknifes", "Throwing knives", tweet)
 tweet = re.sub(r"GodsLove", "God's Love", tweet)
 tweet = re.sub(r"bookboost", "book boost", tweet)
 tweet = re.sub(r"ibooklove", "I book love", tweet)
 tweet = re.sub(r"NestleIndia", "Nestle India", tweet)
 tweet = re.sub(r"realDonaldTrump", "Donald Trump", tweet)
 tweet = re.sub(r"DavidVonderhaar", "David Vonderhaar", tweet)
 tweet = re.sub(r"CecilTheLion", "Cecil The Lion", tweet)
 tweet = re.sub(r"weathernetwork", "weather network", tweet)
 tweet = re.sub(r"GOPDebate", "GOP Debate", tweet)
 tweet = re.sub(r"RickPerry", "Rick Perry", tweet)
 tweet = re.sub(r"frontpage", "front page", tweet)
 tweet = re.sub(r"NewsInTweets", "News In Tweets", tweet)
 tweet = re.sub(r"ViralSpell", "Viral Spell", tweet)
 tweet = re.sub(r"til_now", "until now", tweet)
 tweet = re.sub(r"volcanoinRussia", "volcano in Russia", tweet)
 tweet = re.sub(r"ZippedNews", "Zipped News", tweet)
 tweet = re.sub(r"MicheleBachman", "Michele Bachman", tweet)
 tweet = re.sub(r"53inch", "53 inch", tweet)
 tweet = re.sub(r"KerrickTrial", "Kerrick Trial", tweet)
 tweet = re.sub(r"abstorm", "Alberta Storm", tweet)
 tweet = re.sub(r"Beyhive", "Beyonce hive", tweet)
 tweet = re.sub(r"RockyFire", "Rocky Fire", tweet)
 tweet = re.sub(r"Listen/Buy", "Listen / Buy", tweet)
 tweet = re.sub(r"ArtistsUnited", "Artists United", tweet)
 tweet = re.sub(r"ENGvAUS", "England vs Australia", tweet)
 tweet = re.sub(r"ScottWalker", "Scott Walker", tweet)
 return tweet
total['text'] = total['text'].apply(clean )
x_train = pd.read_csv("/kaggle/input/titanic/train.csv")
 y_train = x_train['Survived']
x_train = x_train.drop(columns=['Survived'])
 x_test = pd.read_csv("/kaggle/input/titanic/test.csv" )<feature_engineering>
tweets = [tweet for tweet in total['text']]
train = total[:len(train)]
test = total[len(train):]
def preprocessing(df):
df["Fare"] =(df["Fare"] - df["Fare"].min())/(df["Fare"].max() - df["Fare"].min())
 df["Fare"] = df["Fare"].fillna(-999)
 df["Sex"] = df["Sex"].factorize() [0]
df["Embarked"] = df["Embarked"].factorize() [0]
for i in range(len(df["Name"])) : df["Name"][i] = df["Name"][i].split(',')[0]
df["Name"] = df["Name"].factorize() [0]
df["Cabin"] = df["Cabin"].factorize() [0]
df["Age"] =(df["Age"] - df["Age"].mean())/ df["Age"].std()
df["Age"] = df["Age"].fillna(-999)
 df["Ticket"] = df["Ticket"].factorize() [0]<categorify>
def generate_ngrams(text, n_gram=1):
token = [token for token in text.lower().split(' ')if token != '' if token not in wordcloud.STOPWORDS]
ngrams = zip(*[token[i:] for i in range(n_gram)])
 return [' '.join(ngram)for ngram in ngrams]
disaster_unigrams = defaultdict(int)
 for word in total[train['target'] == 1]['text']:
for word in generate_ngrams(word, n_gram = 1):
disaster_unigrams[word] += 1
disaster_unigrams = pd.DataFrame(sorted(disaster_unigrams.items() , key=lambda x: x[1])[::-1])
 nondisaster_unigrams = defaultdict(int)
 for word in total[train['target'] == 0]['text']:
for word in generate_ngrams(word, n_gram = 1):
nondisaster_unigrams[word] += 1
nondisaster_unigrams = pd.DataFrame(sorted(nondisaster_unigrams.items() , key=lambda x: x[1])[::-1])
 disaster_bigrams = defaultdict(int)
 for word in total[train['target'] == 1]['text']:
for word in generate_ngrams(word, n_gram = 2):
disaster_bigrams[word] += 1
disaster_bigrams = pd.DataFrame(sorted(disaster_bigrams.items() , key=lambda x: x[1])[::-1])
 nondisaster_bigrams = defaultdict(int)
 for word in total[train['target'] == 0]['text']:
for word in generate_ngrams(word, n_gram = 2):
nondisaster_bigrams[word] += 1
nondisaster_bigrams = pd.DataFrame(sorted(nondisaster_bigrams.items() , key=lambda x: x[1])[::-1])
 disaster_trigrams = defaultdict(int)
 for word in total[train['target'] == 1]['text']:
for word in generate_ngrams(word, n_gram = 3):
disaster_trigrams[word] += 1
disaster_trigrams = pd.DataFrame(sorted(disaster_trigrams.items() , key=lambda x: x[1])[::-1])
 nondisaster_trigrams = defaultdict(int)
 for word in total[train['target'] == 0]['text']:
for word in generate_ngrams(word, n_gram = 3):
nondisaster_trigrams[word] += 1
nondisaster_trigrams = pd.DataFrame(sorted(nondisaster_trigrams.items() , key=lambda x: x[1])[::-1])
 disaster_4grams = defaultdict(int)
 for word in total[train['target'] == 1]['text']:
for word in generate_ngrams(word, n_gram = 4):
disaster_4grams[word] += 1
disaster_4grams = pd.DataFrame(sorted(disaster_4grams.items() , key=lambda x: x[1])[::-1])
 nondisaster_4grams = defaultdict(int)
 for word in total[train['target'] == 0]['text']:
for word in generate_ngrams(word, n_gram = 4):
nondisaster_4grams[word] += 1
nondisaster_4grams = pd.DataFrame(sorted(nondisaster_4grams.items() , key=lambda x: x[1])[::-1] )
x_train_processed = preprocessing(x_train)
 x_test_processed = preprocessing(x_test )<drop_column>
to_exclude = '*+-/() %
[\\]{|}^_`~\t'
to_tokenize = '!"
tokenizer = Tokenizer(filters = to_exclude)
 text = 'Why are you so f%
text = re.sub(r'(['+to_tokenize+'])', r' \1 ', text)
 tokenizer.fit_on_texts([text])
 print(tokenizer.word_index )
x_train_processed = x_train.drop(columns=["Ticket"])
 x_test_processed = x_test.drop(columns=["Ticket"] )<predict_on_test>
model = LogisticRegression(random_state=0, max_iter=2500 ).fit(x_train_processed, y_train)
 pred = model.predict(x_test_processed)
 pred<compute_test_metric>
tokenizer = Tokenizer()
tokenizer.fit_on_texts(tweets)
 sequences = tokenizer.texts_to_sequences(tweets)
 word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
data = pad_sequences(sequences)
 labels = train['target']
print('Shape of data tensor:', data.shape)
 print('Shape of label tensor:', labels.shape)
 nlp_train = data[:len(train)]
labels = labels
nlp_test = data[len(train):]
MAX_SEQUENCE_LENGTH = data.shape[1]
model.score(x_train_processed, y_train )<prepare_output>
embeddings_index = {}
with open('.. /input/glove-global-vectors-for-word-representation/glove.6B.200d.txt','r')as f:
for line in tqdm(f):
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
print('Found %s word vectors in the GloVe library' % len(embeddings_index))
df = pd.DataFrame(pred, columns=["Survived"])
 df.head()<feature_engineering>
EMBEDDING_DIM = 200
df["PassengerId"] = x_test["PassengerId"].values
df<save_to_csv>
embedding_matrix = np.zeros(( len(word_index)+ 1, EMBEDDING_DIM))
for word, i in tqdm(word_index.items()):
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
print("Our embedded matrix is of dimension", embedding_matrix.shape )
df.to_csv('predicts.csv',index=False )<load_from_csv>
embedding = Embedding(len(word_index)+ 1, EMBEDDING_DIM, weights = [embedding_matrix],
input_length = MAX_SEQUENCE_LENGTH, trainable = False)
data_train = pd.read_csv("/kaggle/input/titanic/train.csv")
 data_test = pd.read_csv("/kaggle/input/titanic/test.csv")
 y = data_train.Survived<groupby>
def scale(df, scaler):
return scaler.fit_transform(df.iloc[:, 2:])
 meta_train = scale(train, StandardScaler())
 meta_test = scale(test, StandardScaler() )
data_train.groupby('Sex' ).Survived.mean()<groupby>
def create_lstm(spatial_dropout, dropout, recurrent_dropout, learning_rate, bidirectional = False):
activation = LeakyReLU(alpha = 0.01)
 nlp_input = Input(shape =(MAX_SEQUENCE_LENGTH,), name = 'nlp_input')
 meta_input_train = Input(shape =(7,), name = 'meta_train')
 emb = embedding(nlp_input)
 emb = SpatialDropout1D(dropout )(emb)
 if bidirectional:
nlp_out =(Bidirectional(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal')) )(emb)
 else:
nlp_out =(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal'))(emb)
 x = Concatenate()([nlp_out, meta_input_train])
 x = Dropout(dropout )(x)
 preds = Dense(1, activation='sigmoid', kernel_regularizer = regularizers.l2(1e-4))(x)
 model = Model(inputs=[nlp_input , meta_input_train], outputs = preds)
 optimizer = Adam(learning_rate = learning_rate)
 model.compile(loss = 'binary_crossentropy', optimizer = optimizer, metrics = ['accuracy'])
 return model
data_train.groupby('SibSp' ).Survived.agg(['mean','count'] )<groupby>
lstm = create_lstm(spatial_dropout =.2, dropout =.2, recurrent_dropout =.2,
learning_rate = 3e-4, bidirectional = True)
 lstm.summary()
data_train.groupby('Parch' ).Survived.agg(['mean','count'] )<drop_column>
history1 = lstm.fit([nlp_train, meta_train], labels, validation_split =.2,
epochs = 5, batch_size = 21, verbose = 1 )
X_train = data_train.drop(['Name','Ticket','PassengerId'],axis=1)
 X_test = data_test.drop(['Name','Ticket','PassengerId'],axis=1 )<drop_column>
callback = EarlyStopping(monitor = 'val_loss', patience = 4)
X_train = X_train.drop(['Cabin'],axis=1)
 X_test = X_test.drop(['Cabin'],axis=1 )<filter>
def create_lstm_2(spatial_dropout, dropout, recurrent_dropout, learning_rate, bidirectional = False):
activation = LeakyReLU(alpha = 0.01)
 nlp_input = Input(shape =(MAX_SEQUENCE_LENGTH,), name = 'nlp_input')
 meta_input_train = Input(shape =(7,), name = 'meta_train')
 emb = embedding(nlp_input)
 emb = SpatialDropout1D(dropout )(emb)
 if bidirectional:
nlp_out =(Bidirectional(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal')) )(emb)
 else:
nlp_out =(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal'))(emb)
 x = Concatenate()([nlp_out, meta_input_train])
 x = Dropout(dropout )(x)
 x =(Dense(100, activation = activation, kernel_regularizer = regularizers.l2(1e-4),
kernel_initializer = 'he_normal'))(x)
 x = Dropout(dropout )(x)
 preds = Dense(1, activation='sigmoid', kernel_regularizer = regularizers.l2(1e-4))(x)
 model = Model(inputs=[nlp_input , meta_input_train], outputs = preds)
 optimizer = Adam(learning_rate = learning_rate)
 model.compile(loss = 'binary_crossentropy', optimizer = optimizer, metrics = ['accuracy'])
 return model
X_train[X_train.Embarked.isnull() ]<groupby>
lstm_2 = create_lstm_2(spatial_dropout =.4, dropout =.4, recurrent_dropout =.4,
learning_rate = 3e-4, bidirectional = True)
 lstm_2.summary()
X_train.groupby('Embarked' ).Embarked.count()<data_type_conversions>
history2 = lstm_2.fit([nlp_train, meta_train], labels, validation_split =.2,
epochs = 30, batch_size = 21, verbose = 1 )
X_train.Embarked=X_train.Embarked.fillna('S' )<filter>
submission_lstm = pd.DataFrame()
submission_lstm['id'] = test_id
submission_lstm['prob'] = lstm_2.predict([nlp_test, meta_test])
 submission_lstm['target'] = submission_lstm['prob'].apply(lambda x: 0 if x <.5 else 1)
 submission_lstm.head(10 )
X_train[X_train.Embarked.isnull() ]<categorify>
def create_dual_lstm(spatial_dropout, dropout, recurrent_dropout, learning_rate, bidirectional = False):
activation = LeakyReLU(alpha = 0.01)
 nlp_input = Input(shape =(MAX_SEQUENCE_LENGTH,), name = 'nlp_input')
 meta_input_train = Input(shape =(7,), name = 'meta_train')
 emb = embedding(nlp_input)
 emb = SpatialDropout1D(dropout )(emb)
 if bidirectional:
nlp_out =(Bidirectional(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal', return_sequences = True)) )(emb)
 nlp_out = SpatialDropout1D(dropout )(nlp_out)
 nlp_out =(Bidirectional(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal')) )(emb)
 else:
nlp_out =(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal', return_sequences = True))(emb)
 nlp_out = SpatialDropout1D(dropout )(nlp_out)
 nlp_out =(LSTM(100, dropout = dropout, recurrent_dropout = recurrent_dropout,
kernel_initializer = 'orthogonal'))(emb)
 x = Concatenate()([nlp_out, meta_input_train])
 x = Dropout(dropout )(x)
 preds = Dense(1, activation='sigmoid', kernel_regularizer = regularizers.l2(1e-4))(x)
 model = Model(inputs=[nlp_input , meta_input_train], outputs = preds)
 optimizer = Adam(learning_rate = learning_rate)
 model.compile(loss = 'binary_crossentropy', optimizer = optimizer, metrics = ['accuracy'])
 return model
def impute(cols):
Age = cols[0]
Pclass = cols[1]
if(pd.isnull(Age)) :
if Pclass==1:
return 38
elif Pclass==2:
return 30
else:
return 25
return Age<feature_engineering>
history3 = dual_lstm.fit([nlp_train, meta_train], labels, validation_split =.2,
epochs = 25, batch_size = 21, verbose = 1 )
X_train.Age = X_train[['Age','Pclass']].apply(impute,axis=1)
 X_test.Age = X_test[['Age','Pclass']].apply(impute,axis=1 )<filter>
submission_lstm2 = pd.DataFrame()
submission_lstm2['id'] = test_id
submission_lstm2['prob'] = dual_lstm.predict([nlp_test, meta_test])
 submission_lstm2['target'] = submission_lstm2['prob'].apply(lambda x: 0 if x <.5 else 1)
 submission_lstm2.head(10 )
X_test[X_test.Fare.isnull() ]<feature_engineering>
BATCH_SIZE = 32
EPOCHS = 2
USE_META = True
ADD_DENSE = False
DENSE_DIM = 64
ADD_DROPOUT = False
DROPOUT =.2
X_test['Fare']=X_test['Fare'].fillna(13 )<drop_column>
!pip install --quiet transformers
X_train = X_train.drop(['Survived'],axis=1 )<groupby>
TOKENIZER = AutoTokenizer.from_pretrained("bert-large-uncased")
 enc = TOKENIZER.encode("Encode me!")
 dec = TOKENIZER.decode(enc)
 print("Encode: " + str(enc))
print("Decode: " + str(dec))
X_test.groupby('Sex' ).Sex.count()<import_modules>
def bert_encode(data,maximum_len):
input_ids = []
attention_masks = []
for i in range(len(data.text)) :
encoded = TOKENIZER.encode_plus(data.text[i],
add_special_tokens=True,
max_length=maximum_len,
pad_to_max_length=True,
return_attention_mask=True)
 input_ids.append(encoded['input_ids'])
 attention_masks.append(encoded['attention_mask'])
 return np.array(input_ids),np.array(attention_masks )
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder , LabelEncoder<categorify>
def build_model(model_layer, learning_rate, use_meta = USE_META, add_dense = ADD_DENSE,
dense_dim = DENSE_DIM, add_dropout = ADD_DROPOUT, dropout = DROPOUT):
input_ids = tf.keras.Input(shape=(60,),dtype='int32')
 attention_masks = tf.keras.Input(shape=(60,),dtype='int32')
 meta_input = tf.keras.Input(shape =(meta_train.shape[1],))
transformer_layer = model_layer([input_ids,attention_masks])
 output = transformer_layer[1]
if use_meta:
output = tf.keras.layers.Concatenate()([output, meta_input])
 if add_dense:
print("Training with additional dense layer...")
 output = tf.keras.layers.Dense(dense_dim,activation='relu' )(output)
 if add_dropout:
print("Training with dropout...")
 output = tf.keras.layers.Dropout(dropout )(output)
 output = tf.keras.layers.Dense(1,activation='sigmoid' )(output)
 if use_meta:
print("Training with meta-data...")
 model = tf.keras.models.Model(inputs = [input_ids,attention_masks, meta_input],outputs = output)
 else:
print("Training without meta-data...")
 model = tf.keras.models.Model(inputs = [input_ids,attention_masks],outputs = output)
 model.compile(tf.keras.optimizers.Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy'])
 return model
le =LabelEncoder()
X_train['Sex']=le.fit_transform(X_train['Sex'])
 X_test['Sex'] = le.fit_transform(X_test['Sex'])
 X_train.head()<categorify>
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv' )
onh = OneHotEncoder(handle_unknown='ignore', sparse=False)
 X_train_trans = pd.DataFrame(onh.fit_transform(X_train[['Embarked']]))
X_test_trans = pd.DataFrame(onh.fit_transform(X_test[['Embarked']]))
X_train_trans.index = X_train.index
X_test_trans.index = X_test.index
X_train_conc = X_train.drop(['Embarked'],axis=1)
 X_test_conc = X_test.drop(['Embarked'],axis=1)
 X_train_final = pd.concat([X_train_conc,X_train_trans],axis=1)
 X_test_final = pd.concat([X_test_conc,X_test_trans],axis=1 )<normalization>
bert_large = TFAutoModel.from_pretrained('bert-large-uncased')
 TOKENIZER = AutoTokenizer.from_pretrained("bert-large-uncased")
 train_input_ids,train_attention_masks = bert_encode(train,60)
 test_input_ids,test_attention_masks = bert_encode(test,60)
 print('Train length:', len(train_input_ids))
print('Test length:', len(test_input_ids))
BERT_large = build_model(bert_large, learning_rate = 1e-5)
 BERT_large.summary()
sc= StandardScaler()
X_train_final = sc.fit_transform(X_train_final)
 X_test_final = sc.transform(X_test_final )<train_model>
history_bert = BERT_large.fit([train_input_ids,train_attention_masks, meta_train], train.target,
validation_split =.2, epochs = EPOCHS, callbacks = [checkpoint], batch_size = BATCH_SIZE )
clf = SVC(kernel='rbf', degree = 5)
 clf.fit(X_train_final,y )<save_to_csv>
BERT_large.load_weights('large_model.h5')
 preds_bert = BERT_large.predict([test_input_ids,test_attention_masks,meta_test] )
pred = clf.predict(X_test_final)
 output = pd.DataFrame({'PassengerId':data_test.PassengerId,'Survived':pred})
 output.to_csv('submission.csv',index=False )<set_options>
submission_bert = pd.DataFrame()
submission_bert['id'] = test_id
submission_bert['prob'] = preds_bert
submission_bert['target'] = np.round(submission_bert['prob'] ).astype(int)
 submission_bert.head(10 )
<load_from_csv><EOS> 
submission_bert = submission_bert[['id', 'target']]
submission_bert.to_csv('submission_bert.csv', index = False)
 print('Blended submission has been saved to disk' )