Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<prepare_x_and_y>
%ls /kaggle/input/nlp-getting-started/
y = train_data["Survived"]
features = ["Pclass", "Sex", "Fare", "SibSp", "Parch","FamilySize", "Embarked", "IsAlone"]
X = train_data[features]
X_test = test_data[features]
print(X)
 scoring_method = "f1" 
<train_on_grid>
prefix = '/kaggle/input/nlp-getting-started/'
rf_model = RandomForestClassifier()
rf_params ={
'bootstrap': [True, False],
'max_depth': [10, None],
'max_features': ['auto', 'sqrt'],
'min_samples_leaf': [1, 2, 4],
'min_samples_split': [2, 5, 10],
'n_estimators': [100]}
rf_gs = GridSearchCV(rf_model, rf_params, scoring=scoring_method, cv=8, n_jobs=4)
 rf_gs.fit(X, y)
 print(rf_gs.best_params_)
 print(rf_gs.best_score_ )<compute_train_metric>
train_df = pd.read_csv(prefix + 'train.csv')
cross_val_score(rf_gs, X, y, cv=5 )<save_to_csv>
train_df = pd.DataFrame({
'id':range(len(train_df)) ,
'label':train_df["target"],
'alpha':['a']*train_df.shape[0],
'text': train_df["text"].replace(r'
', ' ', regex=True)
 })
 train_df.head()
len(train_df )
predictions = random_forest.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your pipeline submission was successfully saved!" )<import_modules>
test_df = pd.read_csv(prefix + 'test.csv')
classification_report, f1_score, roc_curve, auc 
<load_from_csv>
test_df.isnull().sum()
PATH = "/kaggle/input/nlp-getting-started/"
train_df = pd.read_csv(f'{PATH}train.csv', low_memory=False)
 train_df.shape<categorify>
test_df = pd.DataFrame({
'id':range(len(test_df)) ,
'label':[0]*test_df.shape[0],
'alpha':['a']*test_df.shape[0],
'text': test_df["text"].replace(r'
', ' ', regex=True)
 })
 test_df.head()
len(test_df )
def clean_text(text):

 cleaned_text = re.sub('<[^>]*>', '', text.lower())
 cleaned_text = re.sub('[\W]+', ' ', cleaned_text)
 cleaned_text = re.sub(r'^https?:\/\/.*[\r
]*', '', cleaned_text)
 emojis = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251]+", flags=re.UNICODE)
 cleaned_text = emojis.sub(r'', cleaned_text)
 return cleaned_text
def split_and_stem(text, stem_text=True, lemmatize_text=False):

 if lemmatize_text:
tokenised = [lemmatizer.lemmatize(word)for word in text.split() ]
elif stem_text:
tokenised = [stemmer.stem(word)for word in text.split() ]
else:
tokenised = [word for word in text.split() ]
return tokenised
def remove_stopwords(text, stem_text=True, lemmatize_text=False):

 words = [word for word in split_and_stem(text, stem_text=stem_text,
lemmatize_text=lemmatize_text)if word not in sw]
words = [word for word in words if len(word)> 1]
new_text = " ".join(words)
 return new_text
def preprocess_text(text, stopwords=True, stem=True, lemmatize=False):

 tokenised = clean_text(text)
 if stopwords:
tokenised = remove_stopwords(tokenised, stem_text=stem, lemmatize_text=lemmatize)
 return tokenised<categorify>
!mkdir data
!pip install contractions
def extensive_clean_and_format(tweet):

 tweet = re.sub(r"\x89Û_", "", tweet)
 tweet = re.sub(r"\x89ÛÒ", "", tweet)
 tweet = re.sub(r"\x89ÛÓ", "", tweet)
 tweet = re.sub(r"\x89ÛÏWhen", "When", tweet)
 tweet = re.sub(r"\x89ÛÏ", "", tweet)
 tweet = re.sub(r"China\x89Ûªs", "China's", tweet)
 tweet = re.sub(r"let\x89Ûªs", "let's", tweet)
 tweet = re.sub(r"\x89Û÷", "", tweet)
 tweet = re.sub(r"\x89Ûª", "", tweet)
 tweet = re.sub(r"\x89Û\x9d", "", tweet)
 tweet = re.sub(r"å_", "", tweet)
 tweet = re.sub(r"\x89Û¢", "", tweet)
 tweet = re.sub(r"\x89Û¢åÊ", "", tweet)
 tweet = re.sub(r"fromåÊwounds", "from wounds", tweet)
 tweet = re.sub(r"åÊ", "", tweet)
 tweet = re.sub(r"åÈ", "", tweet)
 tweet = re.sub(r"JapÌ_n", "Japan", tweet)
 tweet = re.sub(r"Ì©", "e", tweet)
 tweet = re.sub(r"å¨", "", tweet)
 tweet = re.sub(r"SuruÌ¤", "Suruc", tweet)
 tweet = re.sub(r"åÇ", "", tweet)
 tweet = re.sub(r"å£3million", "3 million", tweet)
 tweet = re.sub(r"åÀ", "", tweet)
 tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"Isn't", "is not", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"DON'T", "DO NOT", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"it's", "it is", tweet)
 tweet = re.sub(r"can't", "cannot", tweet)
 tweet = re.sub(r"don't", "do not", tweet)
 tweet = re.sub(r"you're", "you are", tweet)
 tweet = re.sub(r"i've", "I have", tweet)
 tweet = re.sub(r"that's", "that is", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"doesn't", "does not", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"didn't", "did not", tweet)
 tweet = re.sub(r"ain't", "am not", tweet)
 tweet = re.sub(r"you'll", "you will", tweet)
 tweet = re.sub(r"I've", "I have", tweet)
 tweet = re.sub(r"Don't", "do not", tweet)
 tweet = re.sub(r"I'll", "I will", tweet)
 tweet = re.sub(r"I'd", "I would", tweet)
 tweet = re.sub(r"Let's", "Let us", tweet)
 tweet = re.sub(r"you'd", "You would", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"Ain't", "am not", tweet)
 tweet = re.sub(r"Haven't", "Have not", tweet)
 tweet = re.sub(r"Could've", "Could have", tweet)
 tweet = re.sub(r"youve", "you have", tweet)
 tweet = re.sub(r"donå«t", "do not", tweet)
 tweet = re.sub(r"&gt;", ">", tweet)
 tweet = re.sub(r"&lt;", "<", tweet)
 tweet = re.sub(r"&amp;", "&", tweet)
 tweet = re.sub(r"w/e", "whatever", tweet)
 tweet = re.sub(r"w/", "with", tweet)
 tweet = re.sub(r"USAgov", "USA government", tweet)
 tweet = re.sub(r"recentlu", "recently", tweet)
 tweet = re.sub(r"Ph0tos", "Photos", tweet)
 tweet = re.sub(r"amirite", "am I right", tweet)
 tweet = re.sub(r"exp0sed", "exposed", tweet)
 tweet = re.sub(r"<3", "love", tweet)
 tweet = re.sub(r"amageddon", "armageddon", tweet)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"8/5/2015", "2015-08-05", tweet)
 tweet = re.sub(r"WindStorm", "Wind Storm", tweet)
 tweet = re.sub(r"8/6/2015", "2015-08-06", tweet)
 tweet = re.sub(r"10:38PM", "10:38 PM", tweet)
 tweet = re.sub(r"10:30pm", "10:30 PM", tweet)
 tweet = re.sub(r"16yr", "16 year", tweet)
 tweet = re.sub(r"lmao", "laughing my ass off", tweet)
 tweet = re.sub(r"TRAUMATISED", "traumatized", tweet)
 tweet = re.sub(r"IranDeal", "Iran Deal", tweet)
 tweet = re.sub(r"ArianaGrande", "Ariana Grande", tweet)
 tweet = re.sub(r"camilacabello97", "camila cabello", tweet)
 tweet = re.sub(r"RondaRousey", "Ronda Rousey", tweet)
 tweet = re.sub(r"MTVHottest", "MTV Hottest", tweet)
 tweet = re.sub(r"TrapMusic", "Trap Music", tweet)
 tweet = re.sub(r"ProphetMuhammad", "Prophet Muhammad", tweet)
 tweet = re.sub(r"PantherAttack", "Panther Attack", tweet)
 tweet = re.sub(r"StrategicPatience", "Strategic Patience", tweet)
 tweet = re.sub(r"socialnews", "social news", tweet)
 tweet = re.sub(r"NASAHurricane", "NASA Hurricane", tweet)
 tweet = re.sub(r"onlinecommunities", "online communities", tweet)
 tweet = re.sub(r"humanconsumption", "human consumption", tweet)
 tweet = re.sub(r"Typhoon-Devastated", "Typhoon Devastated", tweet)
 tweet = re.sub(r"Meat-Loving", "Meat Loving", tweet)
 tweet = re.sub(r"facialabuse", "facial abuse", tweet)
 tweet = re.sub(r"LakeCounty", "Lake County", tweet)
 tweet = re.sub(r"BeingAuthor", "Being Author", tweet)
 tweet = re.sub(r"withheavenly", "with heavenly", tweet)
 tweet = re.sub(r"thankU", "thank you", tweet)
 tweet = re.sub(r"iTunesMusic", "iTunes Music", tweet)
 tweet = re.sub(r"OffensiveContent", "Offensive Content", tweet)
 tweet = re.sub(r"WorstSummerJob", "Worst Summer Job", tweet)
 tweet = re.sub(r"HarryBeCareful", "Harry Be Careful", tweet)
 tweet = re.sub(r"NASASolarSystem", "NASA Solar System", tweet)
 tweet = re.sub(r"animalrescue", "animal rescue", tweet)
 tweet = re.sub(r"KurtSchlichter", "Kurt Schlichter", tweet)
 tweet = re.sub(r"aRmageddon", "armageddon", tweet)
 tweet = re.sub(r"Throwingknifes", "Throwing knives", tweet)
 tweet = re.sub(r"GodsLove", "God's Love", tweet)
 tweet = re.sub(r"bookboost", "book boost", tweet)
 tweet = re.sub(r"ibooklove", "I book love", tweet)
 tweet = re.sub(r"NestleIndia", "Nestle India", tweet)
 tweet = re.sub(r"realDonaldTrump", "Donald Trump", tweet)
 tweet = re.sub(r"DavidVonderhaar", "David Vonderhaar", tweet)
 tweet = re.sub(r"CecilTheLion", "Cecil The Lion", tweet)
 tweet = re.sub(r"weathernetwork", "weather network", tweet)
 tweet = re.sub(r"withBioterrorism&use", "with Bioterrorism & use", tweet)
 tweet = re.sub(r"Hostage&2", "Hostage & 2", tweet)
 tweet = re.sub(r"GOPDebate", "GOP Debate", tweet)
 tweet = re.sub(r"RickPerry", "Rick Perry", tweet)
 tweet = re.sub(r"frontpage", "front page", tweet)
 tweet = re.sub(r"NewsInTweets", "News In Tweets", tweet)
 tweet = re.sub(r"ViralSpell", "Viral Spell", tweet)
 tweet = re.sub(r"til_now", "until now", tweet)
 tweet = re.sub(r"volcanoinRussia", "volcano in Russia", tweet)
 tweet = re.sub(r"ZippedNews", "Zipped News", tweet)
 tweet = re.sub(r"MicheleBachman", "Michele Bachman", tweet)
 tweet = re.sub(r"53inch", "53 inch", tweet)
 tweet = re.sub(r"KerrickTrial", "Kerrick Trial", tweet)
 tweet = re.sub(r"abstorm", "Alberta Storm", tweet)
 tweet = re.sub(r"Beyhive", "Beyonce hive", tweet)
 tweet = re.sub(r"IDFire", "Idaho Fire", tweet)
 tweet = re.sub(r"DETECTADO", "Detected", tweet)
 tweet = re.sub(r"RockyFire", "Rocky Fire", tweet)
 tweet = re.sub(r"Listen/Buy", "Listen / Buy", tweet)
 tweet = re.sub(r"NickCannon", "Nick Cannon", tweet)
 tweet = re.sub(r"FaroeIslands", "Faroe Islands", tweet)
 tweet = re.sub(r"yycstorm", "Calgary Storm", tweet)
 tweet = re.sub(r"IDPs:", "Internally Displaced People :", tweet)
 tweet = re.sub(r"ArtistsUnited", "Artists United", tweet)
 tweet = re.sub(r"ClaytonBryant", "Clayton Bryant", tweet)
 tweet = re.sub(r"jimmyfallon", "jimmy fallon", tweet)
 tweet = re.sub(r"justinbieber", "justin bieber", tweet)
 tweet = re.sub(r"UTC2015", "UTC 2015", tweet)
 tweet = re.sub(r"Time2015", "Time 2015", tweet)
 tweet = re.sub(r"djicemoon", "dj icemoon", tweet)
 tweet = re.sub(r"LivingSafely", "Living Safely", tweet)
 tweet = re.sub(r"FIFA16", "Fifa 2016", tweet)
 tweet = re.sub(r"thisiswhywecanthavenicethings", "this is why we cannot have nice things", tweet)
 tweet = re.sub(r"bbcnews", "bbc news", tweet)
 tweet = re.sub(r"UndergroundRailraod", "Underground Railraod", tweet)
 tweet = re.sub(r"c4news", "c4 news", tweet)
 tweet = re.sub(r"OBLITERATION", "obliteration", tweet)
 tweet = re.sub(r"MUDSLIDE", "mudslide", tweet)
 tweet = re.sub(r"NoSurrender", "No Surrender", tweet)
 tweet = re.sub(r"NotExplained", "Not Explained", tweet)
 tweet = re.sub(r"greatbritishbakeoff", "great british bake off", tweet)
 tweet = re.sub(r"LondonFire", "London Fire", tweet)
 tweet = re.sub(r"KOTAWeather", "KOTA Weather", tweet)
 tweet = re.sub(r"LuchaUnderground", "Lucha Underground", tweet)
 tweet = re.sub(r"KOIN6News", "KOIN 6 News", tweet)
 tweet = re.sub(r"LiveOnK2", "Live On K2", tweet)
 tweet = re.sub(r"9NewsGoldCoast", "9 News Gold Coast", tweet)
 tweet = re.sub(r"nikeplus", "nike plus", tweet)
 tweet = re.sub(r"david_cameron", "David Cameron", tweet)
 tweet = re.sub(r"peterjukes", "Peter Jukes", tweet)
 tweet = re.sub(r"JamesMelville", "James Melville", tweet)
 tweet = re.sub(r"megynkelly", "Megyn Kelly", tweet)
 tweet = re.sub(r"cnewslive", "C News Live", tweet)
 tweet = re.sub(r"JamaicaObserver", "Jamaica Observer", tweet)
 tweet = re.sub(r"TweetLikeItsSeptember11th2001", "Tweet like it is september 11th 2001", tweet)
 tweet = re.sub(r"cbplawyers", "cbp lawyers", tweet)
 tweet = re.sub(r"fewmoretweets", "few more tweets", tweet)
 tweet = re.sub(r"BlackLivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"cjoyner", "Chris Joyner", tweet)
 tweet = re.sub(r"ENGvAUS", "England vs Australia", tweet)
 tweet = re.sub(r"ScottWalker", "Scott Walker", tweet)
 tweet = re.sub(r"MikeParrActor", "Michael Parr", tweet)
 tweet = re.sub(r"4PlayThursdays", "Foreplay Thursdays", tweet)
 tweet = re.sub(r"TGF2015", "Tontitown Grape Festival", tweet)
 tweet = re.sub(r"realmandyrain", "Mandy Rain", tweet)
 tweet = re.sub(r"GraysonDolan", "Grayson Dolan", tweet)
 tweet = re.sub(r"ApolloBrown", "Apollo Brown", tweet)
 tweet = re.sub(r"saddlebrooke", "Saddlebrooke", tweet)
 tweet = re.sub(r"TontitownGrape", "Tontitown Grape", tweet)
 tweet = re.sub(r"AbbsWinston", "Abbs Winston", tweet)
 tweet = re.sub(r"ShaunKing", "Shaun King", tweet)
 tweet = re.sub(r"MeekMill", "Meek Mill", tweet)
 tweet = re.sub(r"TornadoGiveaway", "Tornado Giveaway", tweet)
 tweet = re.sub(r"GRupdates", "GR updates", tweet)
 tweet = re.sub(r"SouthDowns", "South Downs", tweet)
 tweet = re.sub(r"braininjury", "brain injury", tweet)
 tweet = re.sub(r"auspol", "Australian politics", tweet)
 tweet = re.sub(r"PlannedParenthood", "Planned Parenthood", tweet)
 tweet = re.sub(r"calgaryweather", "Calgary Weather", tweet)
 tweet = re.sub(r"weallheartonedirection", "we all heart one direction", tweet)
 tweet = re.sub(r"edsheeran", "Ed Sheeran", tweet)
 tweet = re.sub(r"TrueHeroes", "True Heroes", tweet)
 tweet = re.sub(r"S3XLEAK", "sex leak", tweet)
 tweet = re.sub(r"ComplexMag", "Complex Magazine", tweet)
 tweet = re.sub(r"TheAdvocateMag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"CityofCalgary", "City of Calgary", tweet)
 tweet = re.sub(r"EbolaOutbreak", "Ebola Outbreak", tweet)
 tweet = re.sub(r"SummerFate", "Summer Fate", tweet)
 tweet = re.sub(r"RAmag", "Royal Academy Magazine", tweet)
 tweet = re.sub(r"offers2go", "offers to go", tweet)
 tweet = re.sub(r"foodscare", "food scare", tweet)
 tweet = re.sub(r"MNPDNashville", "Metropolitan Nashville Police Department", tweet)
 tweet = re.sub(r"TfLBusAlerts", "TfL Bus Alerts", tweet)
 tweet = re.sub(r"GamerGate", "Gamer Gate", tweet)
 tweet = re.sub(r"IHHen", "Humanitarian Relief", tweet)
 tweet = re.sub(r"spinningbot", "spinning bot", tweet)
 tweet = re.sub(r"ModiMinistry", "Modi Ministry", tweet)
 tweet = re.sub(r"TAXIWAYS", "taxi ways", tweet)
 tweet = re.sub(r"Calum5SOS", "Calum Hood", tweet)
 tweet = re.sub(r"po_st", "po.st", tweet)
 tweet = re.sub(r"scoopit", "scoop.it", tweet)
 tweet = re.sub(r"UltimaLucha", "Ultima Lucha", tweet)
 tweet = re.sub(r"JonathanFerrell", "Jonathan Ferrell", tweet)
 tweet = re.sub(r"aria_ahrary", "Aria Ahrary", tweet)
 tweet = re.sub(r"rapidcity", "Rapid City", tweet)
 tweet = re.sub(r"OutBid", "outbid", tweet)
 tweet = re.sub(r"lavenderpoetrycafe", "lavender poetry cafe", tweet)
 tweet = re.sub(r"EudryLantiqua", "Eudry Lantiqua", tweet)
 tweet = re.sub(r"15PM", "15 PM", tweet)
 tweet = re.sub(r"OriginalFunko", "Funko", tweet)
 tweet = re.sub(r"rightwaystan", "Richard Tan", tweet)
 tweet = re.sub(r"CindyNoonan", "Cindy Noonan", tweet)
 tweet = re.sub(r"RT_America", "RT America", tweet)
 tweet = re.sub(r"narendramodi", "Narendra Modi", tweet)
 tweet = re.sub(r"BakeOffFriends", "Bake Off Friends", tweet)
 tweet = re.sub(r"TeamHendrick", "Hendrick Motorsports", tweet)
 tweet = re.sub(r"alexbelloli", "Alex Belloli", tweet)
 tweet = re.sub(r"itsjustinstuart", "Justin Stuart", tweet)
 tweet = re.sub(r"gunsense", "gun sense", tweet)
 tweet = re.sub(r"DebateQuestionsWeWantToHear", "debate questions we want to hear", tweet)
 tweet = re.sub(r"RoyalCarribean", "Royal Carribean", tweet)
 tweet = re.sub(r"samanthaturne19", "Samantha Turner", tweet)
 tweet = re.sub(r"JonVoyage", "Jon Stewart", tweet)
 tweet = re.sub(r"renew911health", "renew 911 health", tweet)
 tweet = re.sub(r"SuryaRay", "Surya Ray", tweet)
 tweet = re.sub(r"pattonoswalt", "Patton Oswalt", tweet)
 tweet = re.sub(r"minhazmerchant", "Minhaz Merchant", tweet)
 tweet = re.sub(r"TLVFaces", "Israel Diaspora Coalition", tweet)
 tweet = re.sub(r"pmarca", "Marc Andreessen", tweet)
 tweet = re.sub(r"pdx911", "Portland Police", tweet)
 tweet = re.sub(r"jamaicaplain", "Jamaica Plain", tweet)
 tweet = re.sub(r"Japton", "Arkansas", tweet)
 tweet = re.sub(r"RouteComplex", "Route Complex", tweet)
 tweet = re.sub(r"INSubcontinent", "Indian Subcontinent", tweet)
 tweet = re.sub(r"NJTurnpike", "New Jersey Turnpike", tweet)
 tweet = re.sub(r"Politifiact", "PolitiFact", tweet)
 tweet = re.sub(r"Hiroshima70", "Hiroshima", tweet)
 tweet = re.sub(r"GMMBC", "Greater Mt Moriah Baptist Church", tweet)
 tweet = re.sub(r"versethe", "verse the", tweet)
 tweet = re.sub(r"TubeStrike", "Tube Strike", tweet)
 tweet = re.sub(r"MissionHills", "Mission Hills", tweet)
 tweet = re.sub(r"ProtectDenaliWolves", "Protect Denali Wolves", tweet)
 tweet = re.sub(r"NANKANA", "Nankana", tweet)
 tweet = re.sub(r"SAHIB", "Sahib", tweet)
 tweet = re.sub(r"PAKPATTAN", "Pakpattan", tweet)
 tweet = re.sub(r"Newz_Sacramento", "News Sacramento", tweet)
 tweet = re.sub(r"gofundme", "go fund me", tweet)
 tweet = re.sub(r"pmharper", "Stephen Harper", tweet)
 tweet = re.sub(r"IvanBerroa", "Ivan Berroa", tweet)
 tweet = re.sub(r"LosDelSonido", "Los Del Sonido", tweet)
 tweet = re.sub(r"bancodeseries", "banco de series", tweet)
 tweet = re.sub(r"timkaine", "Tim Kaine", tweet)
 tweet = re.sub(r"IdentityTheft", "Identity Theft", tweet)
 tweet = re.sub(r"AllLivesMatter", "All Lives Matter", tweet)
 tweet = re.sub(r"mishacollins", "Misha Collins", tweet)
 tweet = re.sub(r"BillNeelyNBC", "Bill Neely", tweet)
 tweet = re.sub(r"BeClearOnCancer", "be clear on cancer", tweet)
 tweet = re.sub(r"Kowing", "Knowing", tweet)
 tweet = re.sub(r"ScreamQueens", "Scream Queens", tweet)
 tweet = re.sub(r"AskCharley", "Ask Charley", tweet)
 tweet = re.sub(r"BlizzHeroes", "Heroes of the Storm", tweet)
 tweet = re.sub(r"BradleyBrad47", "Bradley Brad", tweet)
 tweet = re.sub(r"HannaPH", "Typhoon Hanna", tweet)
 tweet = re.sub(r"meinlcymbals", "MEINL Cymbals", tweet)
 tweet = re.sub(r"Ptbo", "Peterborough", tweet)
 tweet = re.sub(r"cnnbrk", "CNN Breaking News", tweet)
 tweet = re.sub(r"IndianNews", "Indian News", tweet)
 tweet = re.sub(r"savebees", "save bees", tweet)
 tweet = re.sub(r"GreenHarvard", "Green Harvard", tweet)
 tweet = re.sub(r"StandwithPP", "Stand with planned parenthood", tweet)
 tweet = re.sub(r"hermancranston", "Herman Cranston", tweet)
 tweet = re.sub(r"WMUR9", "WMUR-TV", tweet)
 tweet = re.sub(r"RockBottomRadFM", "Rock Bottom Radio", tweet)
 tweet = re.sub(r"ameenshaikh3", "Ameen Shaikh", tweet)
 tweet = re.sub(r"ProSyn", "Project Syndicate", tweet)
 tweet = re.sub(r"Daesh", "ISIS", tweet)
 tweet = re.sub(r"s2g", "swear to god", tweet)
 tweet = re.sub(r"listenlive", "listen live", tweet)
 tweet = re.sub(r"CDCgov", "Centers for Disease Control and Prevention", tweet)
 tweet = re.sub(r"FoxNew", "Fox News", tweet)
 tweet = re.sub(r"CBSBigBrother", "Big Brother", tweet)
 tweet = re.sub(r"JulieDiCaro", "Julie DiCaro", tweet)
 tweet = re.sub(r"theadvocatemag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"RohnertParkDPS", "Rohnert Park Police Department", tweet)
 tweet = re.sub(r"THISIZBWRIGHT", "Bonnie Wright", tweet)
 tweet = re.sub(r"Popularmmos", "Popular MMOs", tweet)
 tweet = re.sub(r"WildHorses", "Wild Horses", tweet)
 tweet = re.sub(r"FantasticFour", "Fantastic Four", tweet)
 tweet = re.sub(r"HORNDALE", "Horndale", tweet)
 tweet = re.sub(r"PINER", "Piner", tweet)
 tweet = re.sub(r"BathAndNorthEastSomerset", "Bath and North East Somerset", tweet)
 tweet = re.sub(r"thatswhatfriendsarefor", "that is what friends are for", tweet)
 tweet = re.sub(r"residualincome", "residual income", tweet)
 tweet = re.sub(r"YahooNewsDigest", "Yahoo News Digest", tweet)
 tweet = re.sub(r"MalaysiaAirlines", "Malaysia Airlines", tweet)
 tweet = re.sub(r"AmazonDeals", "Amazon Deals", tweet)
 tweet = re.sub(r"MissCharleyWebb", "Charley Webb", tweet)
 tweet = re.sub(r"shoalstraffic", "shoals traffic", tweet)
 tweet = re.sub(r"GeorgeFoster72", "George Foster", tweet)
 tweet = re.sub(r"pop2015", "pop 2015", tweet)
 tweet = re.sub(r"_PokemonCards_", "Pokemon Cards", tweet)
 tweet = re.sub(r"DianneG", "Dianne Gallagher", tweet)
 tweet = re.sub(r"KashmirConflict", "Kashmir Conflict", tweet)
 tweet = re.sub(r"BritishBakeOff", "British Bake Off", tweet)
 tweet = re.sub(r"FreeKashmir", "Free Kashmir", tweet)
 tweet = re.sub(r"mattmosley", "Matt Mosley", tweet)
 tweet = re.sub(r"BishopFred", "Bishop Fred", tweet)
 tweet = re.sub(r"EndConflict", "End Conflict", tweet)
 tweet = re.sub(r"EndOccupation", "End Occupation", tweet)
 tweet = re.sub(r"UNHEALED", "unhealed", tweet)
 tweet = re.sub(r"CharlesDagnall", "Charles Dagnall", tweet)
 tweet = re.sub(r"Latestnews", "Latest news", tweet)
 tweet = re.sub(r"KindleCountdown", "Kindle Countdown", tweet)
 tweet = re.sub(r"NoMoreHandouts", "No More Handouts", tweet)
 tweet = re.sub(r"datingtips", "dating tips", tweet)
 tweet = re.sub(r"charlesadler", "Charles Adler", tweet)
 tweet = re.sub(r"twia", "Texas Windstorm Insurance Association", tweet)
 tweet = re.sub(r"txlege", "Texas Legislature", tweet)
 tweet = re.sub(r"WindstormInsurer", "Windstorm Insurer", tweet)
 tweet = re.sub(r"Newss", "News", tweet)
 tweet = re.sub(r"hempoil", "hemp oil", tweet)
 tweet = re.sub(r"CommoditiesAre", "Commodities are", tweet)
 tweet = re.sub(r"tubestrike", "tube strike", tweet)
 tweet = re.sub(r"JoeNBC", "Joe Scarborough", tweet)
 tweet = re.sub(r"LiteraryCakes", "Literary Cakes", tweet)
 tweet = re.sub(r"TI5", "The International 5", tweet)
 tweet = re.sub(r"thehill", "the hill", tweet)
 tweet = re.sub(r"3others", "3 others", tweet)
 tweet = re.sub(r"stighefootball", "Sam Tighe", tweet)
 tweet = re.sub(r"whatstheimportantvideo", "what is the important video", tweet)
 tweet = re.sub(r"ClaudioMeloni", "Claudio Meloni", tweet)
 tweet = re.sub(r"DukeSkywalker", "Duke Skywalker", tweet)
 tweet = re.sub(r"carsonmwr", "Fort Carson", tweet)
 tweet = re.sub(r"offdishduty", "off dish duty", tweet)
 tweet = re.sub(r"andword", "and word", tweet)
 tweet = re.sub(r"rhodeisland", "Rhode Island", tweet)
 tweet = re.sub(r"easternoregon", "Eastern Oregon", tweet)
 tweet = re.sub(r"WAwildfire", "Washington Wildfire", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"57am", "57 am", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"JacobHoggard", "Jacob Hoggard", tweet)
 tweet = re.sub(r"newnewnew", "new new new", tweet)
 tweet = re.sub(r"under50", "under 50", tweet)
 tweet = re.sub(r"getitbeforeitsgone", "get it before it is gone", tweet)
 tweet = re.sub(r"freshoutofthebox", "fresh out of the box", tweet)
 tweet = re.sub(r"amwriting", "am writing", tweet)
 tweet = re.sub(r"Bokoharm", "Boko Haram", tweet)
 tweet = re.sub(r"Nowlike", "Now like", tweet)
 tweet = re.sub(r"seasonfrom", "season from", tweet)
 tweet = re.sub(r"epicente", "epicenter", tweet)
 tweet = re.sub(r"epicenterr", "epicenter", tweet)
 tweet = re.sub(r"sicklife", "sick life", tweet)
 tweet = re.sub(r"yycweather", "Calgary Weather", tweet)
 tweet = re.sub(r"calgarysun", "Calgary Sun", tweet)
 tweet = re.sub(r"approachng", "approaching", tweet)
 tweet = re.sub(r"evng", "evening", tweet)
 tweet = re.sub(r"Sumthng", "something", tweet)
 tweet = re.sub(r"EllenPompeo", "Ellen Pompeo", tweet)
 tweet = re.sub(r"shondarhimes", "Shonda Rhimes", tweet)
 tweet = re.sub(r"ABCNetwork", "ABC Network", tweet)
 tweet = re.sub(r"SushmaSwaraj", "Sushma Swaraj", tweet)
 tweet = re.sub(r"pray4japan", "Pray for Japan", tweet)
 tweet = re.sub(r"hope4japan", "Hope for Japan", tweet)
 tweet = re.sub(r"Illusionimagess", "Illusion images", tweet)
 tweet = re.sub(r"SummerUnderTheStars", "Summer Under The Stars", tweet)
 tweet = re.sub(r"ShallWeDance", "Shall We Dance", tweet)
 tweet = re.sub(r"TCMParty", "TCM Party", tweet)
 tweet = re.sub(r"marijuananews", "marijuana news", tweet)
 tweet = re.sub(r"onbeingwithKristaTippett", "on being with Krista Tippett", tweet)
 tweet = re.sub(r"Beingtweets", "Being tweets", tweet)
 tweet = re.sub(r"newauthors", "new authors", tweet)
 tweet = re.sub(r"remedyyyy", "remedy", tweet)
 tweet = re.sub(r"44PM", "44 PM", tweet)
 tweet = re.sub(r"HeadlinesApp", "Headlines App", tweet)
 tweet = re.sub(r"40PM", "40 PM", tweet)
 tweet = re.sub(r"myswc", "Severe Weather Center", tweet)
 tweet = re.sub(r"ithats", "that is", tweet)
 tweet = re.sub(r"icouldsitinthismomentforever", "I could sit in this moment forever", tweet)
 tweet = re.sub(r"FatLoss", "Fat Loss", tweet)
 tweet = re.sub(r"02PM", "02 PM", tweet)
 tweet = re.sub(r"MetroFmTalk", "Metro Fm Talk", tweet)
 tweet = re.sub(r"Bstrd", "bastard", tweet)
 tweet = re.sub(r"bldy", "bloody", tweet)
 tweet = re.sub(r"MetrofmTalk", "Metro Fm Talk", tweet)
 tweet = re.sub(r"terrorismturn", "terrorism turn", tweet)
 tweet = re.sub(r"BBCNewsAsia", "BBC News Asia", tweet)
 tweet = re.sub(r"BehindTheScenes", "Behind The Scenes", tweet)
 tweet = re.sub(r"GeorgeTakei", "George Takei", tweet)
 tweet = re.sub(r"WomensWeeklyMag", "Womens Weekly Magazine", tweet)
 tweet = re.sub(r"SurvivorsGuidetoEarth", "Survivors Guide to Earth", tweet)
 tweet = re.sub(r"incubusband", "incubus band", tweet)
 tweet = re.sub(r"Babypicturethis", "Baby picture this", tweet)
 tweet = re.sub(r"BombEffects", "Bomb Effects", tweet)
 tweet = re.sub(r"win10", "Windows 10", tweet)
 tweet = re.sub(r"idkidk", "I do not know I do not know", tweet)
 tweet = re.sub(r"TheWalkingDead", "The Walking Dead", tweet)
 tweet = re.sub(r"amyschumer", "Amy Schumer", tweet)
 tweet = re.sub(r"crewlist", "crew list", tweet)
 tweet = re.sub(r"Erdogans", "Erdogan", tweet)
 tweet = re.sub(r"BBCLive", "BBC Live", tweet)
 tweet = re.sub(r"TonyAbbottMHR", "Tony Abbott", tweet)
 tweet = re.sub(r"paulmyerscough", "Paul Myerscough", tweet)
 tweet = re.sub(r"georgegallagher", "George Gallagher", tweet)
 tweet = re.sub(r"JimmieJohnson", "Jimmie Johnson", tweet)
 tweet = re.sub(r"pctool", "pc tool", tweet)
 tweet = re.sub(r"DoingHashtagsRight", "Doing Hashtags Right", tweet)
 tweet = re.sub(r"ThrowbackThursday", "Throwback Thursday", tweet)
 tweet = re.sub(r"SnowBackSunday", "Snowback Sunday", tweet)
 tweet = re.sub(r"LakeEffect", "Lake Effect", tweet)
 tweet = re.sub(r"RTphotographyUK", "Richard Thomas Photography UK", tweet)
 tweet = re.sub(r"BigBang_CBS", "Big Bang CBS", tweet)
 tweet = re.sub(r"writerslife", "writers life", tweet)
 tweet = re.sub(r"NaturalBirth", "Natural Birth", tweet)
 tweet = re.sub(r"UnusualWords", "Unusual Words", tweet)
 tweet = re.sub(r"wizkhalifa", "Wiz Khalifa", tweet)
 tweet = re.sub(r"acreativedc", "a creative DC", tweet)
 tweet = re.sub(r"vscodc", "vsco DC", tweet)
 tweet = re.sub(r"VSCOcam", "vsco camera", tweet)
 tweet = re.sub(r"TheBEACHDC", "The beach DC", tweet)
 tweet = re.sub(r"buildingmuseum", "building museum", tweet)
 tweet = re.sub(r"WorldOil", "World Oil", tweet)
 tweet = re.sub(r"redwedding", "red wedding", tweet)
 tweet = re.sub(r"AmazingRaceCanada", "Amazing Race Canada", tweet)
 tweet = re.sub(r"WakeUpAmerica", "Wake Up America", tweet)
 tweet = re.sub(r"\\Allahuakbar\", "Allahu Akbar", tweet)
 tweet = re.sub(r"bleased", "blessed", tweet)
 tweet = re.sub(r"nigeriantribune", "Nigerian Tribune", tweet)
 tweet = re.sub(r"HIDEO_KOJIMA_EN", "Hideo Kojima", tweet)
 tweet = re.sub(r"FusionFestival", "Fusion Festival", tweet)
 tweet = re.sub(r"50Mixed", "50 Mixed", tweet)
 tweet = re.sub(r"NoAgenda", "No Agenda", tweet)
 tweet = re.sub(r"WhiteGenocide", "White Genocide", tweet)
 tweet = re.sub(r"dirtylying", "dirty lying", tweet)
 tweet = re.sub(r"SyrianRefugees", "Syrian Refugees", tweet)
 tweet = re.sub(r"changetheworld", "change the world", tweet)
 tweet = re.sub(r"Ebolacase", "Ebola case", tweet)
 tweet = re.sub(r"mcgtech", "mcg technologies", tweet)
 tweet = re.sub(r"withweapons", "with weapons", tweet)
 tweet = re.sub(r"advancedwarfare", "advanced warfare", tweet)
 tweet = re.sub(r"letsFootball", "let us Football", tweet)
 tweet = re.sub(r"LateNiteMix", "late night mix", tweet)
 tweet = re.sub(r"PhilCollinsFeed", "Phil Collins", tweet)
 tweet = re.sub(r"RudyHavenstein", "Rudy Havenstein", tweet)
 tweet = re.sub(r"22PM", "22 PM", tweet)
 tweet = re.sub(r"54am", "54 AM", tweet)
 tweet = re.sub(r"38am", "38 AM", tweet)
 tweet = re.sub(r"OldFolkExplainStuff", "Old Folk Explain Stuff", tweet)
 tweet = re.sub(r"BlacklivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"InsaneLimits", "Insane Limits", tweet)
 tweet = re.sub(r"youcantsitwithus", "you cannot sit with us", tweet)
 tweet = re.sub(r"2k15", "2015", tweet)
 tweet = re.sub(r"TheIran", "Iran", tweet)
 tweet = re.sub(r"JimmyFallon", "Jimmy Fallon", tweet)
 tweet = re.sub(r"AlbertBrooks", "Albert Brooks", tweet)
 tweet = re.sub(r"defense_news", "defense news", tweet)
 tweet = re.sub(r"nuclearrcSA", "Nuclear Risk Control Self Assessment", tweet)
 tweet = re.sub(r"Auspol", "Australia Politics", tweet)
 tweet = re.sub(r"NuclearPower", "Nuclear Power", tweet)
 tweet = re.sub(r"WhiteTerrorism", "White Terrorism", tweet)
 tweet = re.sub(r"truthfrequencyradio", "Truth Frequency Radio", tweet)
 tweet = re.sub(r"ErasureIsNotEquality", "Erasure is not equality", tweet)
 tweet = re.sub(r"ProBonoNews", "Pro Bono News", tweet)
 tweet = re.sub(r"JakartaPost", "Jakarta Post", tweet)
 tweet = re.sub(r"toopainful", "too painful", tweet)
 tweet = re.sub(r"melindahaunton", "Melinda Haunton", tweet)
 tweet = re.sub(r"NoNukes", "No Nukes", tweet)
 tweet = re.sub(r"curryspcworld", "Currys PC World", tweet)
 tweet = re.sub(r"ineedcake", "I need cake", tweet)
 tweet = re.sub(r"blackforestgateau", "black forest gateau", tweet)
 tweet = re.sub(r"BBCOne", "BBC One", tweet)
 tweet = re.sub(r"AlexxPage", "Alex Page", tweet)
 tweet = re.sub(r"jonathanserrie", "Jonathan Serrie", tweet)
 tweet = re.sub(r"SocialJerkBlog", "Social Jerk Blog", tweet)
 tweet = re.sub(r"ChelseaVPeretti", "Chelsea Peretti", tweet)
 tweet = re.sub(r"irongiant", "iron giant", tweet)
 tweet = re.sub(r"RonFunches", "Ron Funches", tweet)
 tweet = re.sub(r"TimCook", "Tim Cook", tweet)
 tweet = re.sub(r"sebastianstanisaliveandwell", "Sebastian Stan is alive and well", tweet)
 tweet = re.sub(r"Madsummer", "Mad summer", tweet)
 tweet = re.sub(r"NowYouKnow", "Now you know", tweet)
 tweet = re.sub(r"concertphotography", "concert photography", tweet)
 tweet = re.sub(r"TomLandry", "Tom Landry", tweet)
 tweet = re.sub(r"showgirldayoff", "show girl day off", tweet)
 tweet = re.sub(r"Yougslavia", "Yugoslavia", tweet)
 tweet = re.sub(r"QuantumDataInformatics", "Quantum Data Informatics", tweet)
 tweet = re.sub(r"FromTheDesk", "From The Desk", tweet)
 tweet = re.sub(r"TheaterTrial", "Theater Trial", tweet)
 tweet = re.sub(r"CatoInstitute", "Cato Institute", tweet)
 tweet = re.sub(r"EmekaGift", "Emeka Gift", tweet)
 tweet = re.sub(r"LetsBe_Rational", "Let us be rational", tweet)
 tweet = re.sub(r"Cynicalreality", "Cynical reality", tweet)
 tweet = re.sub(r"FredOlsenCruise", "Fred Olsen Cruise", tweet)
 tweet = re.sub(r"NotSorry", "not sorry", tweet)
 tweet = re.sub(r"UseYourWords", "use your words", tweet)
 tweet = re.sub(r"WordoftheDay", "word of the day", tweet)
 tweet = re.sub(r"Dictionarycom", "Dictionary.com", tweet)
 tweet = re.sub(r"TheBrooklynLife", "The Brooklyn Life", tweet)
 tweet = re.sub(r"jokethey", "joke they", tweet)
 tweet = re.sub(r"nflweek1picks", "NFL week 1 picks", tweet)
 tweet = re.sub(r"uiseful", "useful", tweet)
 tweet = re.sub(r"JusticeDotOrg", "The American Association for Justice", tweet)
 tweet = re.sub(r"autoaccidents", "auto accidents", tweet)
 tweet = re.sub(r"SteveGursten", "Steve Gursten", tweet)
 tweet = re.sub(r"MichiganAutoLaw", "Michigan Auto Law", tweet)
 tweet = re.sub(r"birdgang", "bird gang", tweet)
 tweet = re.sub(r"nflnetwork", "NFL Network", tweet)
 tweet = re.sub(r"NYDNSports", "NY Daily News Sports", tweet)
 tweet = re.sub(r"RVacchianoNYDN", "Ralph Vacchiano NY Daily News", tweet)
 tweet = re.sub(r"EdmontonEsks", "Edmonton Eskimos", tweet)
 tweet = re.sub(r"david_brelsford", "David Brelsford", tweet)
 tweet = re.sub(r"TOI_India", "The Times of India", tweet)
 tweet = re.sub(r"hegot", "he got", tweet)
 tweet = re.sub(r"SkinsOn9", "Skins on 9", tweet)
 tweet = re.sub(r"sothathappened", "so that happened", tweet)
 tweet = re.sub(r"LCOutOfDoors", "LC Out Of Doors", tweet)
 tweet = re.sub(r"NationFirst", "Nation First", tweet)
 tweet = re.sub(r"IndiaToday", "India Today", tweet)
 tweet = re.sub(r"HLPS", "helps", tweet)
 tweet = re.sub(r"HOSTAGESTHROSW", "hostages throw", tweet)
 tweet = re.sub(r"SNCTIONS", "sanctions", tweet)
 tweet = re.sub(r"BidTime", "Bid Time", tweet)
 tweet = re.sub(r"crunchysensible", "crunchy sensible", tweet)
 tweet = re.sub(r"RandomActsOfRomance", "Random acts of romance", tweet)
 tweet = re.sub(r"MomentsAtHill", "Moments at hill", tweet)
 tweet = re.sub(r"eatshit", "eat shit", tweet)
 tweet = re.sub(r"liveleakfun", "live leak fun", tweet)
 tweet = re.sub(r"SahelNews", "Sahel News", tweet)
 tweet = re.sub(r"abc7newsbayarea", "ABC 7 News Bay Area", tweet)
 tweet = re.sub(r"facilitiesmanagement", "facilities management", tweet)
 tweet = re.sub(r"facilitydude", "facility dude", tweet)
 tweet = re.sub(r"CampLogistics", "Camp logistics", tweet)
 tweet = re.sub(r"alaskapublic", "Alaska public", tweet)
 tweet = re.sub(r"MarketResearch", "Market Research", tweet)
 tweet = re.sub(r"AccuracyEsports", "Accuracy Esports", tweet)
 tweet = re.sub(r"TheBodyShopAust", "The Body Shop Australia", tweet)
 tweet = re.sub(r"yychail", "Calgary hail", tweet)
 tweet = re.sub(r"yyctraffic", "Calgary traffic", tweet)
 tweet = re.sub(r"eliotschool", "eliot school", tweet)
 tweet = re.sub(r"TheBrokenCity", "The Broken City", tweet)
 tweet = re.sub(r"OldsFireDept", "Olds Fire Department", tweet)
 tweet = re.sub(r"RiverComplex", "River Complex", tweet)
 tweet = re.sub(r"fieldworksmells", "field work smells", tweet)
 tweet = re.sub(r"IranElection", "Iran Election", tweet)
 tweet = re.sub(r"glowng", "glowing", tweet)
 tweet = re.sub(r"kindlng", "kindling", tweet)
 tweet = re.sub(r"riggd", "rigged", tweet)
 tweet = re.sub(r"slownewsday", "slow news day", tweet)
 tweet = re.sub(r"MyanmarFlood", "Myanmar Flood", tweet)
 tweet = re.sub(r"abc7chicago", "ABC 7 Chicago", tweet)
 tweet = re.sub(r"copolitics", "Colorado Politics", tweet)
 tweet = re.sub(r"AdilGhumro", "Adil Ghumro", tweet)
 tweet = re.sub(r"netbots", "net bots", tweet)
 tweet = re.sub(r"byebyeroad", "bye bye road", tweet)
 tweet = re.sub(r"massiveflooding", "massive flooding", tweet)
 tweet = re.sub(r"EndofUS", "End of United States", tweet)
 tweet = re.sub(r"35PM", "35 PM", tweet)
 tweet = re.sub(r"greektheatrela", "Greek Theatre Los Angeles", tweet)
 tweet = re.sub(r"76mins", "76 minutes", tweet)
 tweet = re.sub(r"publicsafetyfirst", "public safety first", tweet)
 tweet = re.sub(r"livesmatter", "lives matter", tweet)
 tweet = re.sub(r"myhometown", "my hometown", tweet)
 tweet = re.sub(r"tankerfire", "tanker fire", tweet)
 tweet = re.sub(r"MEMORIALDAY", "memorial day", tweet)
 tweet = re.sub(r"MEMORIAL_DAY", "memorial day", tweet)
 tweet = re.sub(r"instaxbooty", "instagram booty", tweet)
 tweet = re.sub(r"Jerusalem_Post", "Jerusalem Post", tweet)
 tweet = re.sub(r"WayneRooney_INA", "Wayne Rooney", tweet)
 tweet = re.sub(r"VirtualReality", "Virtual Reality", tweet)
 tweet = re.sub(r"OculusRift", "Oculus Rift", tweet)
 tweet = re.sub(r"OwenJones84", "Owen Jones", tweet)
 tweet = re.sub(r"jeremycorbyn", "Jeremy Corbyn", tweet)
 tweet = re.sub(r"paulrogers002", "Paul Rogers", tweet)
 tweet = re.sub(r"mortalkombatx", "Mortal Kombat X", tweet)
 tweet = re.sub(r"mortalkombat", "Mortal Kombat", tweet)
 tweet = re.sub(r"FilipeCoelho92", "Filipe Coelho", tweet)
 tweet = re.sub(r"OnlyQuakeNews", "Only Quake News", tweet)
 tweet = re.sub(r"kostumes", "costumes", tweet)
 tweet = re.sub(r"YEEESSSS", "yes", tweet)
 tweet = re.sub(r"ToshikazuKatayama", "Toshikazu Katayama", tweet)
 tweet = re.sub(r"IntlDevelopment", "Intl Development", tweet)
 tweet = re.sub(r"ExtremeWeather", "Extreme Weather", tweet)
 tweet = re.sub(r"WereNotGruberVoters", "We are not gruber voters", tweet)
 tweet = re.sub(r"NewsThousands", "News Thousands", tweet)
 tweet = re.sub(r"EdmundAdamus", "Edmund Adamus", tweet)
 tweet = re.sub(r"EyewitnessWV", "Eye witness WV", tweet)
 tweet = re.sub(r"PhiladelphiaMuseu", "Philadelphia Museum", tweet)
 tweet = re.sub(r"DublinComicCon", "Dublin Comic Con", tweet)
 tweet = re.sub(r"NicholasBrendon", "Nicholas Brendon", tweet)
 tweet = re.sub(r"Alltheway80s", "All the way 80s", tweet)
 tweet = re.sub(r"FromTheField", "From the field", tweet)
 tweet = re.sub(r"NorthIowa", "North Iowa", tweet)
 tweet = re.sub(r"WillowFire", "Willow Fire", tweet)
 tweet = re.sub(r"MadRiverComplex", "Mad River Complex", tweet)
 tweet = re.sub(r"feelingmanly", "feeling manly", tweet)
 tweet = re.sub(r"stillnotoverit", "still not over it", tweet)
 tweet = re.sub(r"FortitudeValley", "Fortitude Valley", tweet)
 tweet = re.sub(r"CoastpowerlineTramTr", "Coast powerline", tweet)
 tweet = re.sub(r"ServicesGold", "Services Gold", tweet)
 tweet = re.sub(r"NewsbrokenEmergency", "News broken emergency", tweet)
 tweet = re.sub(r"Evaucation", "evacuation", tweet)
 tweet = re.sub(r"leaveevacuateexitbe", "leave evacuate exit be", tweet)
 tweet = re.sub(r"P_EOPLE", "PEOPLE", tweet)
 tweet = re.sub(r"Tubestrike", "tube strike", tweet)
 tweet = re.sub(r"CLASS_SICK", "CLASS SICK", tweet)
 tweet = re.sub(r"localplumber", "local plumber", tweet)
 tweet = re.sub(r"awesomejobsiri", "awesome job siri", tweet)
 tweet = re.sub(r"PayForItHow", "Pay for it how", tweet)
 tweet = re.sub(r"ThisIsAfrica", "This is Africa", tweet)
 tweet = re.sub(r"crimeairnetwork", "crime air network", tweet)
 tweet = re.sub(r"KimAcheson", "Kim Acheson", tweet)
 tweet = re.sub(r"cityofcalgary", "City of Calgary", tweet)
 tweet = re.sub(r"prosyndicate", "pro syndicate", tweet)
 tweet = re.sub(r"660NEWS", "660 NEWS", tweet)
 tweet = re.sub(r"BusInsMagazine", "Business Insurance Magazine", tweet)
 tweet = re.sub(r"wfocus", "focus", tweet)
 tweet = re.sub(r"ShastaDam", "Shasta Dam", tweet)
 tweet = re.sub(r"go2MarkFranco", "Mark Franco", tweet)
 tweet = re.sub(r"StephGHinojosa", "Steph Hinojosa", tweet)
 tweet = re.sub(r"Nashgrier", "Nash Grier", tweet)
 tweet = re.sub(r"NashNewVideo", "Nash new video", tweet)
 tweet = re.sub(r"IWouldntGetElectedBecause", "I would not get elected because", tweet)
 tweet = re.sub(r"SHGames", "Sledgehammer Games", tweet)
 tweet = re.sub(r"bedhair", "bed hair", tweet)
 tweet = re.sub(r"JoelHeyman", "Joel Heyman", tweet)
 tweet = re.sub(r"viaYouTube", "via YouTube", tweet)
 tweet = re.sub(r"https?:\/\/t.co\/[A-Za-z0-9]+", "", tweet)
 punctuations = '@
for p in punctuations:
tweet = tweet.replace(p, f' {p} ')
 tweet = tweet.replace('...', '...')
 if '...' not in tweet:
tweet = tweet.replace('.. ', '...')
 tweet = re.sub(r"MH370", "Malaysia Airlines Flight 370", tweet)
 tweet = re.sub(r"mÌ¼sica", "music", tweet)
 tweet = re.sub(r"okwx", "Oklahoma City Weather", tweet)
 tweet = re.sub(r"arwx", "Arkansas Weather", tweet)
 tweet = re.sub(r"gawx", "Georgia Weather", tweet)
 tweet = re.sub(r"scwx", "South Carolina Weather", tweet)
 tweet = re.sub(r"cawx", "California Weather", tweet)
 tweet = re.sub(r"tnwx", "Tennessee Weather", tweet)
 tweet = re.sub(r"azwx", "Arizona Weather", tweet)
 tweet = re.sub(r"alwx", "Alabama Weather", tweet)
 tweet = re.sub(r"wordpressdotcom", "wordpress", tweet)
 tweet = re.sub(r"usNWSgov", "United States National Weather Service", tweet)
 tweet = re.sub(r"Suruc", "Sanliurfa", tweet)
 tweet = re.sub(r"Bestnaijamade", "bestnaijamade", tweet)
 tweet = re.sub(r"SOUDELOR", "Soudelor", tweet)
 return tweet<feature_engineering>
def fix_contractions(text):
return contractions.fix(text)
 def remove_url(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 def remove_mark(text):
table=str.maketrans('','',string.punctuation)
 return text.translate(table)
 print("tweet before contractions fix : ", train_df.iloc[1055]["text"])
 print("-"*20)
 train_df['text']=train_df['text'].apply(lambda x : fix_contractions(x))
test_df['text']=test_df['text'].apply(lambda x : fix_contractions(x))
train_df['text']=train_df['text'].apply(lambda x : remove_url(x))
test_df['text']=test_df['text'].apply(lambda x : remove_url(x))
train_df['text']=train_df['text'].apply(lambda x : remove_mark(x))
test_df['text']=test_df['text'].apply(lambda x : remove_mark(x))
print("tweet after contractions fix : ", train_df.iloc[1055]["text"])
 train_df.to_csv('data/train.tsv', sep='\t', index=False, header=False)
 test_df.to_csv('data/dev.tsv', sep='\t', index=False, header=False )
%%time
stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()
sw = stopwords.words('english')
 sw.append('http')
 sw.append('https')
 sw.append('co')
 sw.append('û_')
 train_df['cleaned text'] = train_df['text'].apply(preprocess_text,
stopwords=True,
stem=True,
lemmatize=False)
 train_df['cleaned text'] = train_df['cleaned text'].apply(extensive_clean_and_format)
 X = train_df['cleaned text'].values
y = train_df['target'].values<string_transform>
logger = logging.getLogger(__name__)
 csv.field_size_limit(2147483647)
 class InputExample(object):

 def __init__(self, guid, text_a, text_b=None, label=None):

 self.guid = guid
self.text_a = text_a
self.text_b = text_b
self.label = label
class InputFeatures(object):

 def __init__(self, input_ids, input_mask, segment_ids, label_id):
self.input_ids = input_ids
self.input_mask = input_mask
self.segment_ids = segment_ids
self.label_id = label_id
class DataProcessor(object):

 def get_train_examples(self, data_dir):

 raise NotImplementedError()
def get_dev_examples(self, data_dir):

 raise NotImplementedError()
def get_labels(self):

 raise NotImplementedError()
@classmethod
def _read_tsv(cls, input_file, quotechar=None):

 with open(input_file, "r", encoding="utf-8-sig")as f:
reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
 lines = []
for line in reader:
if sys.version_info[0] == 2:
line = list(unicode(cell, 'utf-8')for cell in line)
 lines.append(line)
 return lines
class BinaryProcessor(DataProcessor):

 def get_train_examples(self, data_dir):

 return self._create_examples(
 self._read_tsv(os.path.join(data_dir, "train.tsv")) , "train")
 def get_dev_examples(self, data_dir):

 return self._create_examples(
 self._read_tsv(os.path.join(data_dir, "dev.tsv")) , "dev")
 def get_labels(self):

 return ["0", "1"]
def _create_examples(self, lines, set_type):

 examples = []
for(i, line)in enumerate(lines):
guid = "%s-%s" %(set_type, i)
 text_a = line[3]
label = line[1]
examples.append(
 InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
return examples
def convert_example_to_feature(example_row, pad_token=0,
sequence_a_segment_id=0, sequence_b_segment_id=1,
cls_token_segment_id=1, pad_token_segment_id=0,
mask_padding_with_zero=True):
example, label_map, max_seq_length, tokenizer, output_mode, cls_token_at_end, cls_token, sep_token, cls_token_segment_id, pad_on_left, pad_token_segment_id = example_row
tokens_a = tokenizer.tokenize(example.text_a)
 tokens_b = None
if example.text_b:
tokens_b = tokenizer.tokenize(example.text_b)
 _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
 else:
if len(tokens_a)> max_seq_length - 2:
tokens_a = tokens_a[:(max_seq_length - 2)]
tokens = tokens_a + [sep_token]
segment_ids = [sequence_a_segment_id] * len(tokens)
 if tokens_b:
tokens += tokens_b + [sep_token]
segment_ids += [sequence_b_segment_id] *(len(tokens_b)+ 1)
 if cls_token_at_end:
tokens = tokens + [cls_token]
segment_ids = segment_ids + [cls_token_segment_id]
else:
tokens = [cls_token] + tokens
segment_ids = [cls_token_segment_id] + segment_ids
input_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)
 padding_length = max_seq_length - len(input_ids)
 if pad_on_left:
input_ids =([pad_token] * padding_length)+ input_ids
input_mask =([0 if mask_padding_with_zero else 1] * padding_length)+ input_mask
segment_ids =([pad_token_segment_id] * padding_length)+ segment_ids
else:
input_ids = input_ids +([pad_token] * padding_length)
 input_mask = input_mask +([0 if mask_padding_with_zero else 1] * padding_length)
 segment_ids = segment_ids +([pad_token_segment_id] * padding_length)
 assert len(input_ids)== max_seq_length
assert len(input_mask)== max_seq_length
assert len(segment_ids)== max_seq_length
if output_mode == "classification":
label_id = label_map[example.label]
elif output_mode == "regression":
label_id = float(example.label)
 else:
raise KeyError(output_mode)
 return InputFeatures(input_ids=input_ids,
input_mask=input_mask,
segment_ids=segment_ids,
label_id=label_id)
 def convert_examples_to_features(examples, label_list, max_seq_length,
tokenizer, output_mode,
cls_token_at_end=False, pad_on_left=False,
cls_token='[CLS]', sep_token='[SEP]', pad_token=0,
sequence_a_segment_id=0, sequence_b_segment_id=1,
cls_token_segment_id=1, pad_token_segment_id=0,
mask_padding_with_zero=True):

 label_map = {label : i for i, label in enumerate(label_list)}
examples = [(example, label_map, max_seq_length, tokenizer, output_mode, cls_token_at_end, cls_token, sep_token, cls_token_segment_id, pad_on_left, pad_token_segment_id)for example in examples]
process_count = 1
with Pool(process_count)as p:
features = list(tqdm(p.imap(convert_example_to_feature, examples, chunksize=100), total=len(examples)))
 return features
def _truncate_seq_pair(tokens_a, tokens_b, max_length):

 while True:
total_length = len(tokens_a)+ len(tokens_b)
 if total_length <= max_length:
break
if len(tokens_a)> len(tokens_b):
tokens_a.pop()
else:
tokens_b.pop()
processors = {
"binary": BinaryProcessor
}
output_modes = {
"binary": "classification"
}
GLUE_TASKS_NUM_LABELS = {
"binary": 2
}
def create_corpus(text_data):

 corpus = []
for sentence in text_data:
for word in sentence.split() :
corpus.append(word)
 return corpus
def top_words(text_corpus, top_n=25):

 def_dict = defaultdict(int)
 for word in text_corpus:
def_dict[word] += 1
most_common = sorted(def_dict.items() , key=lambda x : x[1], reverse=True)[:top_n]
return most_common<load_from_csv>
!pip install pytorch_transformers
PATH = "/kaggle/input/nlp-getting-started/"
test_df = pd.read_csv(f'{PATH}test.csv', low_memory=False)
 test_df['cleaned_text'] = test_df['text'].apply(preprocess_text,
stopwords=True,
stem=True,
lemmatize=False)
 test_df['cleaned_text'] = test_df['cleaned_text'].apply(extensive_clean_and_format)
 X_test = test_df['cleaned_text'].values
X_test.shape<split>
TensorDataset)
 XLMConfig, XLMForSequenceClassification, XLMTokenizer,
XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer,
RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer)
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__ )
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.2, random_state=0)
 print("Shapes of our data:
X_train: {0}
y_train: {1}
X_val: {2}
y_val: {3} ".format(X_train.shape,
y_train.shape,
X_val.shape,
y_val.shape))<feature_engineering>
args = {
'data_dir': './data/',
'model_type': 'bert',
'model_name': 'bert-base-cased',
'task_name': 'binary',
'output_dir': 'outputs/',
'cache_dir': 'cache/',
'do_train': True,
'do_eval': True,
'fp16': False,
'fp16_opt_level': 'O1',
'max_seq_length': 128,
'output_mode': 'classification',
'train_batch_size': 8,
'eval_batch_size': 8,
'gradient_accumulation_steps': 1,
'num_train_epochs': 3,
'weight_decay': 0,
'learning_rate': 4e-5,
'adam_epsilon': 1e-8,
'warmup_steps': 0,
'max_grad_norm': 1.0,
'logging_steps': 50,
'evaluate_during_training': False,
'save_steps': 2000,
'eval_all_checkpoints': True,
'overwrite_output_dir': False,
'reprocess_input_data': True,
'notes': 'Using Yelp Reviews dataset'
}
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )
n_grams =(1,3)
 vectorizer = TfidfVectorizer(analyzer='word', ngram_range=n_grams)
 X_train_vec = vectorizer.fit_transform(X_train)
 X_val_vec = vectorizer.transform(X_val)
 X_test_vec = vectorizer.transform(X_test )<compute_train_metric>
MODEL_CLASSES = {
'bert':(BertConfig, BertForSequenceClassification, BertTokenizer),
'xlnet':(XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer),
'xlm':(XLMConfig, XLMForSequenceClassification, XLMTokenizer),
'roberta':(RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer)
 }
config_class, model_class, tokenizer_class = MODEL_CLASSES[args['model_type']]
def multi_model_cross_validation(clf_tuple_list, X, y, K_folds=10, score_type='accuracy', random_seed=0):

 model_names, model_scores = [], []
for name, model in clf_list:
k_fold = StratifiedKFold(n_splits=K_folds, shuffle=True, random_state=random_seed)
 cross_val_results = cross_val_score(model, X, y, cv=k_fold, scoring=score_type, n_jobs=-1)
 model_names.append(name)
 model_scores.append(cross_val_results)
 print("{0:<40} {1:.5f} +/- {2:.5f}".format(name, cross_val_results.mean() , cross_val_results.std()))
return model_names, model_scores
def boxplot_comparison(model_names, model_scores, figsize=(12, 6), score_type="Accuracy",
title="Sentiment Analysis Classification Comparison"):

 fig = plt.figure(figsize=figsize)
 fig.suptitle(title, fontsize=18)
 ax = fig.add_subplot(111)
 sns.boxplot(x=model_names, y=model_scores)
 ax.set_xticklabels(model_names)
 ax.set_xlabel("Model", fontsize=16)
 ax.set_ylabel("Model Score({})".format(score_type), fontsize=16)
 ax.set_xticklabels(ax.get_xticklabels() , rotation=60)
 plt.show()
return
clf_list = [("Perceptron", Perceptron(eta0=0.1)) , 
("Logistic Regression", LogisticRegression(C=10.0)) , 
("Support Vector Machine", SVC(kernel='linear', C=1.0)) , 
("Decision Tree", DecisionTreeClassifier()), 
("Random Forest", RandomForestClassifier(n_estimators=10)) , 
("Multinomial Naive Bayes", MultinomialNB()), 
("Ridge Classifier", RidgeClassifier())]
model_names, model_scores = multi_model_cross_validation(clf_list, X_train_vec, y_train)
 boxplot_comparison(model_names, model_scores )<compute_train_metric>
config = config_class.from_pretrained(args['model_name'], num_labels=2, finetuning_task=args['task_name'])
 tokenizer = tokenizer_class.from_pretrained(args['model_name'] )
def test_set_performances(clf_tuple_list, X_train, y_train, X_test,
y_test, score_type='accuracy', print_results=True):

 model_names, model_accuracies, model_f1 = [], [], []
if print_results:
print("{0:<30} {1:<10} {2:<10}
{3}".format("Model", "Accuracy",
"F1-Score", "-"*50))
for name, model in clf_list:
model.fit(X_train, y_train)
 y_preds = model.predict(X_test)
 accuracy = accuracy_score(y_test, y_preds)
 test_f1 = f1_score(y_test, y_preds, average='macro')
 model_names.append(name)
 model_accuracies.append(accuracy)
 model_f1.append(test_f1)
 if print_results:
print("{0:<30} {1:<10.5f} {2:<10.5f}".format(name, accuracy, test_f1))
return model_names, model_accuracies, model_f1
model_names, test_acc, test_f1 = test_set_performances(clf_list, X_train_vec, y_train, X_val_vec, y_val )<train_model>
model = model_class.from_pretrained(args['model_name'] )
svc_clf = SVC(kernel='linear', C=1.0, probability=True)
 svc_clf.fit(X_train_vec, y_train )<prepare_x_and_y>
model.to(device )
X = train_df['cleaned text'].values
y = train_df['target'].values<feature_engineering>
task = args['task_name']
processor = processors[task]()
label_list = processor.get_labels()
num_labels = len(label_list )
vectorizer = TfidfVectorizer(analyzer='word', ngram_range=n_grams)
 X_vec = vectorizer.fit_transform(X)
 X_test_vec = vectorizer.transform(X_test )<train_model>
def load_and_cache_examples(task, tokenizer, evaluate=False):
processor = processors[task]()
output_mode = args['output_mode']
mode = 'dev' if evaluate else 'train'
cached_features_file = os.path.join(args['data_dir'], f"cached_{mode}_{args['model_name']}_{args['max_seq_length']}_{task}")
 if os.path.exists(cached_features_file)and not args['reprocess_input_data']:
logger.info("Loading features from cached file %s", cached_features_file)
 features = torch.load(cached_features_file)
 else:
logger.info("Creating features from dataset file at %s", args['data_dir'])
 label_list = processor.get_labels()
examples = processor.get_dev_examples(args['data_dir'])if evaluate else processor.get_train_examples(args['data_dir'])
 features = convert_examples_to_features(examples, label_list, args['max_seq_length'], tokenizer, output_mode,
cls_token_at_end=bool(args['model_type'] in ['xlnet']),
cls_token=tokenizer.cls_token,
sep_token=tokenizer.sep_token,
cls_token_segment_id=2 if args['model_type'] in ['xlnet'] else 0,
pad_on_left=bool(args['model_type'] in ['xlnet']),
pad_token_segment_id=4 if args['model_type'] in ['xlnet'] else 0)
 logger.info("Saving features into cached file %s", cached_features_file)
 torch.save(features, cached_features_file)
 all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
 all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
 all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
 if output_mode == "classification":
all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long)
 elif output_mode == "regression":
all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.float)
 dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
 return dataset
svc_clf = SVC(kernel='linear', C=1.0)
 svc_clf.fit(X_vec, y )<predict_on_test>
def train(train_dataset, model, tokenizer):
tb_writer = SummaryWriter()
train_sampler = RandomSampler(train_dataset)
 train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args['train_batch_size'])
 t_total = len(train_dataloader)// args['gradient_accumulation_steps'] * args['num_train_epochs']
no_decay = ['bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args['weight_decay']},
{'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
optimizer = AdamW(optimizer_grouped_parameters, lr=args['learning_rate'], eps=args['adam_epsilon'])
 scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args['warmup_steps'], t_total=t_total)
 if args['fp16']:
try:
except ImportError:
raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
 model, optimizer = amp.initialize(model, optimizer, opt_level=args['fp16_opt_level'])
 logger.info("***** Running training *****")
 logger.info(" Num examples = %d", len(train_dataset))
logger.info(" Num Epochs = %d", args['num_train_epochs'])
 logger.info(" Total train batch size = %d", args['train_batch_size'])
 logger.info(" Gradient Accumulation steps = %d", args['gradient_accumulation_steps'])
 logger.info(" Total optimization steps = %d", t_total)
 global_step = 0
tr_loss, logging_loss = 0.0, 0.0
model.zero_grad()
train_iterator = trange(int(args['num_train_epochs']), desc="Epoch")
 for _ in train_iterator:
epoch_iterator = tqdm_notebook(train_dataloader, desc="Iteration")
 for step, batch in enumerate(epoch_iterator):
model.train()
batch = tuple(t.to(device)for t in batch)
 inputs = {'input_ids': batch[0],
'attention_mask': batch[1],
'token_type_ids': batch[2] if args['model_type'] in ['bert', 'xlnet'] else None,
'labels': batch[3]}
outputs = model(input_ids=batch[0],attention_mask=batch[1],token_type_ids=batch[2],labels=batch[3])
 loss = outputs[0]
print("\r%f" % loss, end='')
 if args['gradient_accumulation_steps'] > 1:
loss = loss / args['gradient_accumulation_steps']
if args['fp16']:
with amp.scale_loss(loss, optimizer)as scaled_loss:
scaled_loss.backward()
torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args['max_grad_norm'])
 else:
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , args['max_grad_norm'])
 tr_loss += loss.item()
if(step + 1)% args['gradient_accumulation_steps'] == 0:
scheduler.step()
optimizer.step()
model.zero_grad()
global_step += 1
if args['logging_steps'] > 0 and global_step % args['logging_steps'] == 0:
if args['evaluate_during_training']:
results = evaluate(model, tokenizer)
 for key, value in results.items() :
tb_writer.add_scalar('eval_{}'.format(key), value, global_step)
 tb_writer.add_scalar('lr', scheduler.get_lr() [0], global_step)
 tb_writer.add_scalar('loss',(tr_loss - logging_loss)/args['logging_steps'], global_step)
 logging_loss = tr_loss
if args['save_steps'] > 0 and global_step % args['save_steps'] == 0:
output_dir = os.path.join(args['output_dir'], 'checkpoint-{}'.format(global_step))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
 model_to_save = model.module if hasattr(model, 'module')else model
model_to_save.save_pretrained(output_dir)
 logger.info("Saving model checkpoint to %s", output_dir)
 return global_step, tr_loss / global_step
test_preds = svc_clf.predict(X_test_vec)
 test_df['target'] = test_preds<save_to_csv>
if args['do_train']:
train_dataset = load_and_cache_examples(task, tokenizer)
 global_step, tr_loss = train(train_dataset, model, tokenizer)
 logger.info(" global_step = %s, average loss = %s", global_step, tr_loss )
submission = test_df.loc[:, ['id', 'target']]
submission.to_csv('SVC_submission.csv', index=False)
 submission.head()<train_model>
def submit(pred_ids):
sub = pd.read_csv(prefix+'sample_submission.csv')
 sub['target'] = list(map(int,pred_ids))
sub.to_csv('submission.csv', index=False )
clf_list = [("Logistic Regression", LogisticRegression(C=10.0)) , 
("Support Vector Machine", SVC(kernel='linear', C=1.0, probability=True)) , 
("Random Forest", RandomForestClassifier(n_estimators=500)) , 
("Multinomial Naive Bayes", MultinomialNB())]
ensemble_clf = VotingClassifier(estimators=clf_list, voting='soft')
 ensemble_clf.fit(X_vec, y )<save_to_csv>
def get_mismatched(labels, preds):
mismatched = labels != preds
examples = processor.get_dev_examples(args['data_dir'])
 wrong = [i for(i, v)in zip(examples, mismatched)if v]
return wrong
def get_eval_report(labels, preds):
mcc = matthews_corrcoef(labels, preds)
 tn, fp, fn, tp = confusion_matrix(labels, preds ).ravel()
return {
"mcc": mcc,
"tp": tp,
"tn": tn,
"fp": fp,
"fn": fn
}, get_mismatched(labels, preds)
 def compute_metrics(task_name, preds, labels):
assert len(preds)== len(labels)
 return get_eval_report(labels, preds)
 def evaluate(model, tokenizer, prefix=""):
eval_output_dir = args['output_dir']
results = {}
EVAL_TASK = args['task_name']
eval_dataset = load_and_cache_examples(EVAL_TASK, tokenizer, evaluate=True)
 if not os.path.exists(eval_output_dir):
os.makedirs(eval_output_dir)
 eval_sampler = SequentialSampler(eval_dataset)
 eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args['eval_batch_size'])
 preds_ = []
logger.info("***** Running evaluation {} *****".format(prefix))
logger.info(" Num examples = %d", len(eval_dataset))
logger.info(" Batch size = %d", args['eval_batch_size'])
 eval_loss = 0.0
nb_eval_steps = 0
preds = None
out_label_ids = None
preds_ = []
for batch in tqdm_notebook(eval_dataloader, desc="Evaluating"):
model.eval()
batch = tuple(t.to(device)for t in batch)
 with torch.no_grad() :
inputs = {'input_ids': batch[0],
'attention_mask': batch[1],
'token_type_ids': batch[2] if args['model_type'] in ['bert', 'xlnet'] else None,
'labels': batch[3]}
outputs = model(**inputs)
 tmp_eval_loss, logits = outputs[:2]
for i in range(logits.size(0)) :
preds_.append(logits[i,:].cpu().max(0)[1].item())
 eval_loss += tmp_eval_loss.mean().item()
nb_eval_steps += 1
if preds is None:
preds = logits.detach().cpu().numpy()
out_label_ids = inputs['labels'].detach().cpu().numpy()
else:
preds = np.append(preds, logits.detach().cpu().numpy() , axis=0)
 out_label_ids = np.append(out_label_ids, inputs['labels'].detach().cpu().numpy() , axis=0)
 submit(preds_)
 eval_loss = eval_loss / nb_eval_steps
if args['output_mode'] == "classification":
preds = np.argmax(preds, axis=1)
 elif args['output_mode'] == "regression":
preds = np.squeeze(preds)
 result, wrong = compute_metrics(EVAL_TASK, preds, out_label_ids)
 results.update(result)
 output_eval_file = os.path.join(eval_output_dir, "eval_results.txt")
 with open(output_eval_file, "w")as writer:
logger.info("***** Eval results {} *****".format(prefix))
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s
" %(key, str(result[key])))
 return results, wrong
<load_from_csv><EOS>