Kaggle competition: Natural Language Processing with Disaster Tweets
piv = pd.DataFrame(ids[:, 0], columns=["id"] )<feature_engineering>
print(tf.__version__ )
QUANTILES = ["0.005", "0.025", "0.165", "0.250", "0.500", "0.750", "0.835", "0.975", "0.995"]
VALID = []
EVAL = []
for i, quantile in tqdm(enumerate(QUANTILES)) :
t1 = pd.DataFrame(pv[:,:, i]*sv, columns=names)
 t1 = piv.join(t1)
 t1["id"] = t1["id"] + f"_{quantile}_validation"
t2 = pd.DataFrame(pe[:,:, i]*se, columns=names)
 t2 = piv.join(t2)
 t2["id"] = t2["id"] + f"_{quantile}_evaluation"
VALID.append(t1)
 EVAL.append(t2)
<create_dataframe>
train_pd = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test_pd = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )
sub = pd.DataFrame()
sub = sub.append(VALID + EVAL)
 del VALID, EVAL, t1, t2<save_to_csv>
no_of_rows, no_of_cols = train_pd.shape
sub.to_csv("submission.csv", index=False )<import_modules>
print("keyword column analysis")
 print("-" * 30)
 no_na_keywords, _ = train_pd[train_pd['keyword'].isna() ].shape
print(f"number of rows w/ value: NaN: {no_na_keywords} i.e {(no_na_keywords/no_of_rows)*100}%")
 print(f"number of rows w/0 value: NaN: {no_of_rows - no_na_keywords} i.e {(( no_of_rows-no_na_keywords)/no_of_rows)*100}%")
 print()
print('*' * 80)
 print()
print("location column analysis")
 print("-" * 30)
 no_na_location, _ = train_pd[train_pd['location'].isna() ].shape
print(f"number of rows w/ value: NaN: {no_na_location} i.e {(no_na_location/no_of_rows)*100}%")
 print(f"number of rows w/0 value: NaN: {no_of_rows - no_na_location} i.e.{(( no_of_rows-no_na_location)/no_of_rows)*100}%" )
import os
import sys
import operator
import numpy as np
import pandas as pd
from scipy import sparse
import xgboost as xgb
from sklearn import model_selection, preprocessing, ensemble
from sklearn.metrics import log_loss
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer<train_model>
train_pd = train_pd.drop(['location'], axis=1)
 train_pd = train_pd[train_pd['keyword'].notna() ]
train_pd = train_pd.reset_index(drop=True)
 print(train_pd.head())
 print(f"Number of rows: {len(train_pd)}" )
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0, num_rounds=1000):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.1
param['max_depth'] = 6
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = 1
param['subsample'] = 0.7
param['colsample_bytree'] = 0.7
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=20)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 return pred_test_y, model<load_from_disk>
test_pd = test_pd.drop(['location'], axis=1)
 test_pd = test_pd.fillna(value={'keyword': ''})
 print(test_pd.head())
 print(f"Number of rows: {len(test_pd)}" )
data_path = ".. /input/"
train_file = data_path + "train.json"
test_file = data_path + "test.json"
train_df = pd.read_json(train_file)
 test_df = pd.read_json(test_file)
 print(train_df.shape)
 print(test_df.shape )<define_variables>
eng_stopwords = set(stopwords.words('english'))
stemmer = PorterStemmer()
features_to_use = ["bathrooms", "bedrooms", "latitude", "longitude", "price"]<feature_engineering>
def parse_tweet(tweet):
tweet = tweet.lstrip().rstrip()
tweet = re.sub("http:\S+", "", tweet)
 tweet = re.sub("https:\S+", "", tweet)
 tweet = re.sub(r"[,.;
tweet = tweet.replace(" ", " ")
 tweet = tweet.lower()
tweet = tweet.replace('...', '...' ).strip()
tweet = tweet.replace("'", " ' " ).strip()
tweet = re.sub(r"\x89Û_", "", tweet)
 tweet = re.sub(r"\x89ÛÒ", "", tweet)
 tweet = re.sub(r"\x89ÛÓ", "", tweet)
 tweet = re.sub(r"\x89ÛÏWhen", "When", tweet)
 tweet = re.sub(r"\x89ÛÏ", "", tweet)
 tweet = re.sub(r"China\x89Ûªs", "China's", tweet)
 tweet = re.sub(r"let\x89Ûªs", "let's", tweet)
 tweet = re.sub(r"\x89Û÷", "", tweet)
 tweet = re.sub(r"\x89Ûª", "", tweet)
 tweet = re.sub(r"\x89Û\x9d", "", tweet)
 tweet = re.sub(r"å_", "", tweet)
 tweet = re.sub(r"\x89Û¢", "", tweet)
 tweet = re.sub(r"\x89Û¢åÊ", "", tweet)
 tweet = re.sub(r"fromåÊwounds", "from wounds", tweet)
 tweet = re.sub(r"åÊ", "", tweet)
 tweet = re.sub(r"åÈ", "", tweet)
 tweet = re.sub(r"JapÌ_n", "Japan", tweet)
 tweet = re.sub(r"Ì©", "e", tweet)
 tweet = re.sub(r"å¨", "", tweet)
 tweet = re.sub(r"SuruÌ¤", "Suruc", tweet)
 tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"Isn't", "is not", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"DON'T", "DO NOT", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"&gt;", ">", tweet)
 tweet = re.sub(r"&lt;", "<", tweet)
 tweet = re.sub(r"&amp;", "&", tweet)
 tweet = re.sub(r"w/e", "whatever", tweet)
 tweet = re.sub(r"w/", "with", tweet)
 tweet = re.sub(r"USAgov", "USA government", tweet)
 tweet = re.sub(r"recentlu", "recently", tweet)
 tweet = re.sub(r"Ph0tos", "Photos", tweet)
 tweet = re.sub(r"exp0sed", "exposed", tweet)
 tweet = re.sub(r"<3", "love", tweet)
 tweet = re.sub(r"amageddon", "armageddon", tweet)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"8/5/2015", "2015-08-05", tweet)
 tweet = re.sub(r"chest/torso", "chest / torso", tweet)
 tweet = re.sub(r"WindStorm", "Wind Storm", tweet)
 tweet = re.sub(r"8/6/2015", "2015-08-06", tweet)
 tweet = re.sub(r"10:38PM", "10:38 PM", tweet)
 tweet = re.sub(r"10:30pm", "10:30 PM", tweet)
 tweet = re.sub(r"MH370:", "MH370 :", tweet)
 tweet = re.sub(r"PM:", "Prime Minister :", tweet)
 tweet = re.sub(r"Legionnaires:", "Legionnaires :", tweet)
 tweet = re.sub(r"Latest:", "Latest :", tweet)
 tweet = re.sub(r"Crash:", "Crash :", tweet)
 tweet = re.sub(r"News:", "News :", tweet)
 tweet = re.sub(r"derailment:", "derailment :", tweet)
 tweet = re.sub(r"attack:", "attack :", tweet)
 tweet = re.sub(r"Saipan:", "Saipan :", tweet)
 tweet = re.sub(r"Photo:", "Photo :", tweet)
 tweet = re.sub(r"Funtenna:", "Funtenna :", tweet)
 tweet = re.sub(r"quiz:", "quiz :", tweet)
 tweet = re.sub(r"VIDEO:", "VIDEO :", tweet)
 tweet = re.sub(r"MP:", "MP :", tweet)
 tweet = re.sub(r"UTC2015-08-05", "UTC 2015-08-05", tweet)
 tweet = re.sub(r"California:", "California :", tweet)
 tweet = re.sub(r"horror:", "horror :", tweet)
 tweet = re.sub(r"Past:", "Past :", tweet)
 tweet = re.sub(r"Time2015-08-06", "Time 2015-08-06", tweet)
 tweet = re.sub(r"here:", "here :", tweet)
 tweet = re.sub(r"fires.", "fires.", tweet)
 tweet = re.sub(r"Forest:", "Forest :", tweet)
 tweet = re.sub(r"Cramer:", "Cramer :", tweet)
 tweet = re.sub(r"Chile:", "Chile :", tweet)
 tweet = re.sub(r"link:", "link :", tweet)
 tweet = re.sub(r"crash:", "crash :", tweet)
 tweet = re.sub(r"Video:", "Video :", tweet)
 tweet = re.sub(r"Bestnaijamade:", "bestnaijamade :", tweet)
 tweet = re.sub(r"NWS:", "National Weather Service :", tweet)
 tweet = re.sub(r".caught", ".caught", tweet)
 tweet = re.sub(r"Hobbit:", "Hobbit :", tweet)
 tweet = re.sub(r"2015:", "2015 :", tweet)
 tweet = re.sub(r"post:", "post :", tweet)
 tweet = re.sub(r"BREAKING:", "BREAKING :", tweet)
 tweet = re.sub(r"Island:", "Island :", tweet)
 tweet = re.sub(r"Med:", "Med :", tweet)
 tweet = re.sub(r"97/Georgia", "97 / Georgia", tweet)
 tweet = re.sub(r"Here:", "Here :", tweet)
 tweet = re.sub(r"horror;", "horror ;", tweet)
 tweet = re.sub(r"people;", "people ;", tweet)
 tweet = re.sub(r"refugees;", "refugees ;", tweet)
 tweet = re.sub(r"Genocide;", "Genocide ;", tweet)
 tweet = re.sub(r".POTUS", ".POTUS", tweet)
 tweet = re.sub(r"Collision-No", "Collision - No", tweet)
 tweet = re.sub(r"Rear-", "Rear -", tweet)
 tweet = re.sub(r"Broadway:", "Broadway :", tweet)
 tweet = re.sub(r"Correction:", "Correction :", tweet)
 tweet = re.sub(r"UPDATE:", "UPDATE :", tweet)
 tweet = re.sub(r"Times:", "Times :", tweet)
 tweet = re.sub(r"RT:", "RT :", tweet)
 tweet = re.sub(r"Police:", "Police :", tweet)
 tweet = re.sub(r"Training:", "Training :", tweet)
 tweet = re.sub(r"Hawaii:", "Hawaii :", tweet)
 tweet = re.sub(r"Selfies:", "Selfies :", tweet)
 tweet = re.sub(r"Content:", "Content :", tweet)
 tweet = re.sub(r"101:", "101 :", tweet)
 tweet = re.sub(r"story:", "story :", tweet)
 tweet = re.sub(r"injured:", "injured :", tweet)
 tweet = re.sub(r"poll:", "poll :", tweet)
 tweet = re.sub(r"Guide:", "Guide :", tweet)
 tweet = re.sub(r"Update:", "Update :", tweet)
 tweet = re.sub(r"alarm:", "alarm :", tweet)
 tweet = re.sub(r"floods:", "floods :", tweet)
 tweet = re.sub(r"Flood:", "Flood :", tweet)
 tweet = re.sub(r"MH370;", "MH370 ;", tweet)
 tweet = re.sub(r"life:", "life :", tweet)
 tweet = re.sub(r"crush:", "crush :", tweet)
 tweet = re.sub(r"now:", "now :", tweet)
 tweet = re.sub(r"Vote:", "Vote :", tweet)
 tweet = re.sub(r"Catastrophe.", "Catastrophe.", tweet)
 tweet = re.sub(r"library:", "library :", tweet)
 tweet = re.sub(r"Bush:", "Bush :", tweet)
 tweet = re.sub(r";ACCIDENT", "; ACCIDENT", tweet)
 tweet = re.sub(r"accident:", "accident :", tweet)
 tweet = re.sub(r"Taiwan;", "Taiwan ;", tweet)
 tweet = re.sub(r"Map:", "Map :", tweet)
 tweet = re.sub(r"failure:", "failure :", tweet)
 tweet = re.sub(r"150-Foot", "150 - Foot", tweet)
 tweet = re.sub(r"failure:", "failure :", tweet)
 tweet = re.sub(r"prefer:", "prefer :", tweet)
 tweet = re.sub(r"CNN:", "CNN :", tweet)
 tweet = re.sub(r"Oops:", "Oops :", tweet)
 tweet = re.sub(r"Disco:", "Disco :", tweet)
 tweet = re.sub(r"Disease:", "Disease :", tweet)
 tweet = re.sub(r"Grows:", "Grows :", tweet)
 tweet = re.sub(r"projected:", "projected :", tweet)
 tweet = re.sub(r"Pakistan.", "Pakistan.", tweet)
 tweet = re.sub(r"ministers:", "ministers :", tweet)
 tweet = re.sub(r"Photos:", "Photos :", tweet)
 tweet = re.sub(r"Disease:", "Disease :", tweet)
 tweet = re.sub(r"pres:", "press :", tweet)
 tweet = re.sub(r"winds.", "winds.", tweet)
 tweet = re.sub(r"MPH.", "MPH.", tweet)
 tweet = re.sub(r"PHOTOS:", "PHOTOS :", tweet)
 tweet = re.sub(r"Time2015-08-05", "Time 2015-08-05", tweet)
 tweet = re.sub(r"Denmark:", "Denmark :", tweet)
 tweet = re.sub(r"Articles:", "Articles :", tweet)
 tweet = re.sub(r"Crash:", "Crash :", tweet)
 tweet = re.sub(r"casualties.:", "casualties.:", tweet)
 tweet = re.sub(r"Afghanistan:", "Afghanistan :", tweet)
 tweet = re.sub(r"Day:", "Day :", tweet)
 tweet = re.sub(r"AVERTED:", "AVERTED :", tweet)
 tweet = re.sub(r"sitting:", "sitting :", tweet)
 tweet = re.sub(r"Multiplayer:", "Multiplayer :", tweet)
 tweet = re.sub(r"Kaduna:", "Kaduna :", tweet)
 tweet = re.sub(r"favorite:", "favorite :", tweet)
 tweet = re.sub(r"home:", "home :", tweet)
 tweet = re.sub(r"just:", "just :", tweet)
 tweet = re.sub(r"Collision-1141", "Collision - 1141", tweet)
 tweet = re.sub(r"County:", "County :", tweet)
 tweet = re.sub(r"Duty:", "Duty :", tweet)
 tweet = re.sub(r"page:", "page :", tweet)
 tweet = re.sub(r"Attack:", "Attack :", tweet)
 tweet = re.sub(r"Minecraft:", "Minecraft :", tweet)
 tweet = re.sub(r"wounds;", "wounds ;", tweet)
 tweet = re.sub(r"Shots:", "Shots :", tweet)
 tweet = re.sub(r"shots:", "shots :", tweet)
 tweet = re.sub(r"Gunfire:", "Gunfire :", tweet)
 tweet = re.sub(r"hike:", "hike :", tweet)
 tweet = re.sub(r"Email:", "Email :", tweet)
 tweet = re.sub(r"System:", "System :", tweet)
 tweet = re.sub(r"Radio:", "Radio :", tweet)
 tweet = re.sub(r"King:", "King :", tweet)
 tweet = re.sub(r"upheaval:", "upheaval :", tweet)
 tweet = re.sub(r"tragedy;", "tragedy ;", tweet)
 tweet = re.sub(r"HERE:", "HERE :", tweet)
 tweet = re.sub(r"terrorism:", "terrorism :", tweet)
 tweet = re.sub(r"police:", "police :", tweet)
 tweet = re.sub(r"Mosque:", "Mosque :", tweet)
 tweet = re.sub(r"Rightways:", "Rightways :", tweet)
 tweet = re.sub(r"Brooklyn:", "Brooklyn :", tweet)
 tweet = re.sub(r"Arrived:", "Arrived :", tweet)
 tweet = re.sub(r"Home:", "Home :", tweet)
 tweet = re.sub(r"Earth:", "Earth :", tweet)
 tweet = re.sub(r"three:", "three :", tweet)
 tweet = re.sub(r"IranDeal", "Iran Deal", tweet)
 tweet = re.sub(r"ArianaGrande", "Ariana Grande", tweet)
 tweet = re.sub(r"camilacabello97", "camila cabello", tweet)
 tweet = re.sub(r"RondaRousey", "Ronda Rousey", tweet)
 tweet = re.sub(r"MTVHottest", "MTV Hottest", tweet)
 tweet = re.sub(r"TrapMusic", "Trap Music", tweet)
 tweet = re.sub(r"ProphetMuhammad", "Prophet Muhammad", tweet)
 tweet = re.sub(r"PantherAttack", "Panther Attack", tweet)
 tweet = re.sub(r"StrategicPatience", "Strategic Patience", tweet)
 tweet = re.sub(r"socialnews", "social news", tweet)
 tweet = re.sub(r"NASAHurricane", "NASA Hurricane", tweet)
 tweet = re.sub(r"onlinecommunities", "online communities", tweet)
 tweet = re.sub(r"humanconsumption", "human consumption", tweet)
 tweet = re.sub(r"Typhoon-Devastated", "Typhoon Devastated", tweet)
 tweet = re.sub(r"Meat-Loving", "Meat Loving", tweet)
 tweet = re.sub(r"facialabuse", "facial abuse", tweet)
 tweet = re.sub(r"LakeCounty", "Lake County", tweet)
 tweet = re.sub(r"BeingAuthor", "Being Author", tweet)
 tweet = re.sub(r"withheavenly", "with heavenly", tweet)
 tweet = re.sub(r"thankU", "thank you", tweet)
 tweet = re.sub(r"iTunesMusic", "iTunes Music", tweet)
 tweet = re.sub(r"OffensiveContent", "Offensive Content", tweet)
 tweet = re.sub(r"WorstSummerJob", "Worst Summer Job", tweet)
 tweet = re.sub(r"HarryBeCareful", "Harry Be Careful", tweet)
 tweet = re.sub(r"NASASolarSystem", "NASA Solar System", tweet)
 tweet = re.sub(r"animalrescue", "animal rescue", tweet)
 tweet = re.sub(r"KurtSchlichter", "Kurt Schlichter", tweet)
 tweet = re.sub(r"aRmageddon", "armageddon", tweet)
 tweet = re.sub(r"Throwingknifes", "Throwing knives", tweet)
 tweet = re.sub(r"GodsLove", "God's Love", tweet)
 tweet = re.sub(r"bookboost", "book boost", tweet)
 tweet = re.sub(r"ibooklove", "I book love", tweet)
 tweet = re.sub(r"NestleIndia", "Nestle India", tweet)
 tweet = re.sub(r"realDonaldTrump", "Donald Trump", tweet)
 tweet = re.sub(r"DavidVonderhaar", "David Vonderhaar", tweet)
 tweet = re.sub(r"CecilTheLion", "Cecil The Lion", tweet)
 tweet = re.sub(r"weathernetwork", "weather network", tweet)
 tweet = re.sub(r"withBioterrorism&use", "with Bioterrorism & use", tweet)
 tweet = re.sub(r"Hostage&2", "Hostage & 2", tweet)
 tweet = re.sub(r"GOPDebate", "GOP Debate", tweet)
 tweet = re.sub(r"RickPerry", "Rick Perry", tweet)
 tweet = re.sub(r"frontpage", "front page", tweet)
 tweet = re.sub(r"NewsInTweets", "News In Tweets", tweet)
 tweet = re.sub(r"ViralSpell", "Viral Spell", tweet)
 tweet = re.sub(r"til_now", "until now", tweet)
 tweet = re.sub(r"volcanoinRussia", "volcano in Russia", tweet)
 tweet = re.sub(r"ZippedNews", "Zipped News", tweet)
 tweet = re.sub(r"MicheleBachman", "Michele Bachman", tweet)
 tweet = re.sub(r"53inch", "53 inch", tweet)
 tweet = re.sub(r"KerrickTrial", "Kerrick Trial", tweet)
 tweet = re.sub(r"abstorm", "Alberta Storm", tweet)
 tweet = re.sub(r"Beyhive", "Beyonce hive", tweet)
 tweet = re.sub(r"IDFire", "Idaho Fire", tweet)
 tweet = re.sub(r"DETECTADO", "Detected", tweet)
 tweet = re.sub(r"RockyFire", "Rocky Fire", tweet)
 tweet = re.sub(r"Listen/Buy", "Listen / Buy", tweet)
 tweet = re.sub(r"NickCannon", "Nick Cannon", tweet)
 tweet = re.sub(r"FaroeIslands", "Faroe Islands", tweet)
 tweet = re.sub(r"yycstorm", "Calgary Storm", tweet)
 tweet = re.sub(r"IDPs:", "Internally Displaced People :", tweet)
 tweet = re.sub(r"ArtistsUnited", "Artists United", tweet)
 tweet = re.sub(r"ClaytonBryant", "Clayton Bryant", tweet)
 tweet = re.sub(r"jimmyfallon", "jimmy fallon", tweet)
 words = [x.lstrip().rstrip() for x in tweet.split() ]
words = [x for x in words if x and x not in eng_stopwords]
if not words:
return ""
words = ['user' if x.startswith('@')else x for x in words]
tweet = " ".join(words)
 tweet = re.sub(r"[,.;@
return tweet
train_df["num_photos"] = train_df["photos"].apply(len)
 test_df["num_photos"] = test_df["photos"].apply(len)
 train_df["num_features"] = train_df["features"].apply(len)
 test_df["num_features"] = test_df["features"].apply(len)
 train_df["num_description_words"] = train_df["description"].apply(lambda x: len(x.split(" ")))
 test_df["num_description_words"] = test_df["description"].apply(lambda x: len(x.split(" ")))
 train_df["created"] = pd.to_datetime(train_df["created"])
 test_df["created"] = pd.to_datetime(test_df["created"])
 train_df["created_year"] = train_df["created"].dt.year
test_df["created_year"] = test_df["created"].dt.year
train_df["created_month"] = train_df["created"].dt.month
test_df["created_month"] = test_df["created"].dt.month
train_df["created_day"] = train_df["created"].dt.day
test_df["created_day"] = test_df["created"].dt.day
train_df["created_hour"] = train_df["created"].dt.hour
test_df["created_hour"] = test_df["created"].dt.hour
features_to_use.extend(["num_photos", "num_features", "num_description_words","created_year", "created_month", "created_day", "listing_id", "created_hour"] )<categorify>
train_pd['combined_filtered_data'] = train_pd['keyword'] + ' ' + train_pd['text']
train_pd['combined_filtered_data'] = train_pd['combined_filtered_data'].apply(lambda x: parse_tweet(x))
train_pd.head()
categorical = ["display_address", "manager_id", "building_id", "street_address"]
for f in categorical:
if train_df[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[f].values)+ list(test_df[f].values))
train_df[f] = lbl.transform(list(train_df[f].values))
test_df[f] = lbl.transform(list(test_df[f].values))
features_to_use.append(f )<feature_engineering>
test_pd['combined_filtered_data'] = test_pd['keyword'] + ' ' + test_pd['text']
test_pd['combined_filtered_data'] = test_pd['combined_filtered_data'].apply(lambda x: parse_tweet(x))
test_pd.head()
train_df['features'] = train_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
test_df['features'] = test_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
print(train_df["features"].head())
 tfidf = CountVectorizer(stop_words='english', max_features=200)
 tr_sparse = tfidf.fit_transform(train_df["features"])
 te_sparse = tfidf.transform(test_df["features"] )<prepare_x_and_y>
test_pd = test_pd.fillna(value={'combined_filtered_data': ''} )
train_X = sparse.hstack([train_df[features_to_use], tr_sparse] ).tocsr()
test_X = sparse.hstack([test_df[features_to_use], te_sparse] ).tocsr()
target_num_map = {'high':0, 'medium':1, 'low':2}
train_y = np.array(train_df['interest_level'].apply(lambda x: target_num_map[x]))
print(train_X.shape, test_X.shape )<compute_train_metric>
max_len = max(map(len, train_pd['combined_filtered_data']))
max_len = max(max_len, max(map(len, test_pd['combined_filtered_data'])))
 print(max_len )
cv_scores = []
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2016)
 for dev_index, val_index in kf.split(range(train_X.shape[0])) :
dev_X, val_X = train_X[dev_index,:], train_X[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
preds, model = runXGB(dev_X, dev_y, val_X, val_y)
 cv_scores.append(log_loss(val_y, preds))
print(cv_scores)
 break<save_to_csv>
embedding_dim = 300
trunc_type='post'
padding_type='post'
oov_tok = "<OOV>"
preds, model = runXGB(train_X, train_y, test_X, num_rounds=400)
 out_df = pd.DataFrame(preds)
 out_df.columns = ["high", "medium", "low"]
out_df["listing_id"] = test_df.listing_id.values
out_df.to_csv("xgb_starter2.csv", index=False)
<set_options>
tokenizer = Tokenizer(oov_token=oov_tok)
 tokenizer.fit_on_texts(train_pd['combined_filtered_data'])
 tokenizer.fit_on_texts(test_pd['combined_filtered_data'])
 word_index = tokenizer.word_index
vocab_size = len(word_index )
physical_devices = tf.config.list_physical_devices('GPU')
 try:
tf.config.experimental.set_memory_growth(physical_devices[0], True)
 except:
pass 
<import_modules>
X = tokenizer.texts_to_sequences(train_pd['combined_filtered_data'])
 X_padded = pad_sequences(X, maxlen=max_len, padding=padding_type, truncating=trunc_type )
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]<split>
split = int(1.0 * len(X_padded))
trainX, trainY = X_padded[:split], train_pd['target'][:split]
valX, valY = X_padded[split:], train_pd['target'][split:]
class CVTuner(kt.engine.tuner.Tuner):
def run_trial(self, trial, X, y, splits, batch_size=32, epochs=1,callbacks=None):
val_losses = []
for train_indices, test_indices in splits:
X_train, X_test = [x[train_indices] for x in X], [x[test_indices] for x in X]
y_train, y_test = [a[train_indices] for a in y], [a[test_indices] for a in y]
if len(X_train)< 2:
X_train = X_train[0]
X_test = X_test[0]
if len(y_train)< 2:
y_train = y_train[0]
y_test = y_test[0]
model = self.hypermodel.build(trial.hyperparameters)
 hist = model.fit(X_train,y_train,
validation_data=(X_test,y_test),
epochs=epochs,
batch_size=batch_size,
callbacks=callbacks)
 val_losses.append([hist.history[k][-1] for k in hist.history])
 val_losses = np.asarray(val_losses)
 self.oracle.update_trial(trial.trial_id, {k:np.mean(val_losses[:,i])for i,k in enumerate(hist.history.keys())})
 self.save_model(trial.trial_id, model )<load_from_csv>
testX = tokenizer.texts_to_sequences(test_pd['combined_filtered_data'])
 testX = pad_sequences(testX, maxlen=max_len, padding=padding_type, truncating=trunc_type )
TRAINING = False
USE_FINETUNE = True
FOLDS = 5
SEED = 42
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0.00001)&(train['resp_2'] > 0.00001)&(train['resp_3'] > 0.00001)&(train['resp_4'] > 0.00001)&(train['resp'] > 0.00001)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0.000001 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )<categorify>
embeddings_glove = np.load('.. /input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl', allow_pickle=True )
def create_autoencoder(input_dim,output_dim,noise=0.05):
i = Input(input_dim)
 encoded = BatchNormalization()(i)
 encoded = GaussianNoise(noise )(encoded)
 encoded = Dense(640,activation='relu' )(encoded)
 decoded = Dropout(0.2 )(encoded)
 decoded = Dense(input_dim,name='decoded' )(decoded)
 x = Dense(320,activation='relu' )(decoded)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 encoder = Model(inputs=i,outputs=encoded)
 autoencoder = Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=Adam(0.001),loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder, encoder<categorify>
embedding_matrix = np.zeros(( vocab_size + 1, embedding_dim))
for word, i in word_index.items() :
embedding_vector = embeddings_glove.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
def create_model(hp,input_dim,output_dim,encoder):
inputs = Input(input_dim)
 x = encoder(inputs)
 x = Concatenate()([x,inputs])
 x = BatchNormalization()(x)
 x = Dropout(hp.Float('init_dropout',0.0,0.5))(x)
 for i in range(hp.Int('num_layers',1,5)) :
x = Dense(hp.Int('num_units_{i}',128,256))(x)
 x = BatchNormalization()(x)
 x = Lambda(tf.keras.activations.swish )(x)
 x = Dropout(hp.Float(f'dropout_{i}',0.0,0.5))(x)
 x = Dense(output_dim,activation='sigmoid' )(x)
 model = Model(inputs=inputs,outputs=x)
 model.compile(optimizer=Adam(hp.Float('lr',0.00001,0.1,default=0.001)) ,loss=BinaryCrossentropy(label_smoothing=hp.Float('label_smoothing',0.0,0.1)) ,metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model<load_pretrained>
class F1Score(tf.keras.metrics.Metric):
def __init__(self, name='f1_score', **kwargs):
super(F1Score, self ).__init__(name=name, **kwargs)
 self.p = tf.keras.metrics.Precision()
self.r = tf.keras.metrics.Recall()
def update_state(self, *args, **kwargs):
self.p.update_state(*args, **kwargs)
 self.r.update_state(*args, **kwargs)
 def reset_states(self):
self.p.reset_states()
self.r.reset_states()
def result(self):
p_res, r_res = self.p.result() , self.r.result()
return(2 * p_res * r_res)/(p_res + r_res )
pd.read_pickle(f'.. /input/vae005/best_hp_42.pkl' )<categorify>
def create_model() :
tf.keras.backend.clear_session()
model = tf.keras.Sequential([
tf.keras.layers.Embedding(
 vocab_size + 1, embedding_dim,
weights=[embedding_matrix], input_length=max_len,
trainable=False 
),
tf.keras.layers.Bidirectional(
 tf.keras.layers.LSTM(300)
),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(30, activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(1, activation='sigmoid')
 ])
 model.compile(
 loss='binary_crossentropy',
optimizer=tf.keras.optimizers.Adam() ,
metrics=[F1Score() ] 
)
 return model
autoencoder, encoder = create_autoencoder(X.shape[-1],y.shape[-1],noise=0.1)
 if TRAINING:
autoencoder.fit(X,(X,y),
epochs=1002,
batch_size=16384,
validation_split=0.1,
callbacks=[EarlyStopping('val_loss',patience=10,restore_best_weights=True)])
 encoder.save_weights('./encoder.hdf5')
 else:
encoder.load_weights('.. /input/vae005/encoder.hdf5')
 encoder.trainable = False<categorify>
final_predictions = np.average(predictions, axis=0)
 test_pd['target'] = final_predictions
test_pd['target'] = test_pd['target'].apply(lambda x: 1 if x >= 0.5 else 0 )
<predict_on_test><EOS> 
test_pd.to_csv('/kaggle/working/submission.csv', columns=['id', 'target'], index=False )