Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<normalization>
nltk.download('stopwords')
var_thresh = VarianceThreshold(0.9)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]<define_variables>
warnings.filterwarnings('ignore')
 stop_words = set(stopwords.words("english"))
stemmer_snowball = SnowballStemmer("english")
 stemmer_porter = PorterStemmer()
plt.style.use('ggplot')
 sns.set(font_scale=1.5)
 pd.options.display.max_columns = 250
pd.options.display.max_rows = 250
q_list = [0.1, 0.2, 0.25, 0.3, 0.4, 0.5, 0.6, 0.7, 0.75, 0.8, 0.9]<feature_engineering>
Max_length = 42
Dropout_num = 0
learning_rate = 6e-6
valid = 0.2
epochs_num = 3
batch_size_num = 16
ids_error_corrected = True
def make_knn_features(features, pre_name='G'):
q_list = [0.1, 0.2, 0.25, 0.3, 0.4, 0.5, 0.6, 0.7, 0.75, 0.8, 0.9]
kNN_features = {}
for q in q_list:
kNN_features[f'q-{q}'] = features.quantile(q, axis=1 ).values
kNN_features['min'] = features.min(axis=1 ).values
kNN_features['max'] = features.max(axis=1 ).values
kNN_features['std'] = features.std(axis=1 ).values
kNN_features['mean'] = features.mean(axis=1 ).values
return pd.DataFrame(kNN_features, columns=kNN_features.keys() )<feature_engineering>
train_path = "/kaggle/input/nlp-getting-started/train.csv"
test_path = "/kaggle/input/nlp-getting-started/test.csv"
train_df = pd.read_csv(train_path, dtype={'id': np.int16, 'target': np.int8})
 test_df = pd.read_csv(test_path, dtype={'id': np.int16})
 submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 print(f'train: {train_df.shape}')
 print(f'test: {test_df.shape}' )
TRAIN_MASK =(train_features['cp_type'] != 'ctl_vehicle')
 TEST_MASK =(test_features['cp_type'] != 'ctl_vehicle')
 g_knn_train_features = make_knn_features(train_features[GENES][TRAIN_MASK])
 g_knn_test_features = make_knn_features(test_features[GENES][TEST_MASK])
 c_knn_train_features = make_knn_features(train_features[CELLS][TRAIN_MASK])
 c_knn_test_features = make_knn_features(test_features[CELLS][TEST_MASK])
 knn_train_features = pd.concat([g_knn_train_features, c_knn_train_features], axis=1)
 knn_test_features = pd.concat([g_knn_test_features, c_knn_test_features], axis=1 )<drop_column>
train_df.isnull().sum()
target_cols = train_targets_scored.drop('sig_id', axis=1 ).columns.values.tolist()<train_model>
train_df = train_df.drop(['location', 'keyword'], axis=1)
 test_df = test_df.drop(['location', 'keyword'], axis=1 )
clfr = RandomForestClassifier(10)
 clfr.fit(X=knn_train_features, y=train_targets_scored[target_cols][train_features['cp_type']!='ctl_vehicle'] )<predict_on_test>
def remove_url(tweet):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',tweet)
 def remove_html(tweet):
html=re.compile(r'<.*?>')
 return html.sub(r'',tweet)
 def remove_emoji(tweet):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', tweet )
clfr_res = clfr.predict(knn_train_features )<filter>
def special_characters(tweet):
tweet = re.sub(r"SuruÌ¤", "Suruc", tweet)
 tweet = re.sub(r"JapÌ_n", "Japan", tweet)
 tweet = re.sub(r"\x89ÛÏWhen", "When", tweet)
 tweet = re.sub(r"å£3million", "3 million", tweet)
 tweet = re.sub(r"fromåÊwounds", "from wounds", tweet)
 tweet = re.sub(r"mÌ¼sica", "music", tweet)
 tweet = re.sub(r"donå«t", "do not", tweet)
 tweet = re.sub(r"didn`t", "did not", tweet)
 tweet = re.sub(r"i\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"i\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"i\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"let\x89Ûªs", "let us", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"that\x89Ûªs", "that is", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"here\x89Ûªs", "here is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"you\x89Ûªre", "you are", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"You\x89Ûªve", "You have", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"You\x89Ûªll", "You will", tweet)
 tweet = re.sub(r"China\x89Ûªs", "China's", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"\x89Û_", "", tweet)
 tweet = re.sub(r"\x89Û¢", "", tweet)
 tweet = re.sub(r"\x89ÛÒ", "", tweet)
 tweet = re.sub(r"\x89ÛÓ", "", tweet)
 tweet = re.sub(r"\x89ÛÏ", "", tweet)
 tweet = re.sub(r"\x89Û÷", "", tweet)
 tweet = re.sub(r"\x89Ûª", "", tweet)
 tweet = re.sub(r"\x89Û¢åÊ", "", tweet)
 tweet = re.sub(r"\x89Û\x9d", "", tweet)
 tweet = re.sub(r"å_", "", tweet)
 tweet = re.sub(r"å¨", "", tweet)
 tweet = re.sub(r"åÀ", "", tweet)
 tweet = re.sub(r"åÇ", "", tweet)
 tweet = re.sub(r"åÊ", "", tweet)
 tweet = re.sub(r"åÈ", "", tweet)
 tweet = re.sub(r"Ì©", "", tweet)
 tweet = re.sub(r"&lt;", "<", tweet)
 tweet = re.sub(r"&gt;", ">", tweet)
 tweet = re.sub(r"&amp;", "&", tweet)
 return tweet
def remove_nonASCII(tweet):
tweet = ''.join([x for x in tweet if x in string.printable])
 return tweet
( clfr_res == train_targets_scored[target_cols][train_features['cp_type']!='ctl_vehicle'].values ).mean()<create_dataframe>
def expand_contractions(tweet):
tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"i'M", "I am", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"I'd", "I would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"I'll", "I will", tweet)
 tweet = re.sub(r"i've", "I have", tweet)
 tweet = re.sub(r"I've", "I have", tweet)
 tweet = re.sub(r"you're", "you are", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"you'd", "you would", tweet)
 tweet = re.sub(r"You'd", "You would", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"You've", "You have", tweet)
 tweet = re.sub(r"you'll", "you will", tweet)
 tweet = re.sub(r"You'll", "You will", tweet)
 tweet = re.sub(r"y'know", "you know", tweet)
 tweet = re.sub(r"Y'know", "You know", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"We'd", "We would", tweet)
 tweet = re.sub(r"WE'VE", "We have", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"We'll", "We will", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"They'd", "They would", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"They've", "They have", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"They'll", "They will", tweet)
 tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"He'll", "He will", tweet)
 tweet = re.sub(r"she's", "she is", tweet)
 tweet = re.sub(r"She's", "She is", tweet)
 tweet = re.sub(r"she'll", "she will", tweet)
 tweet = re.sub(r"She'll", "She will", tweet)
 tweet = re.sub(r"it's", "it is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"It'll", "It will", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"Isn't", "Is not", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"Who's", "Who is", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"that's", "that is", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"here's", "here is", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Where's", "Where is", tweet)
 tweet = re.sub(r"wHeRE's", "where is", tweet)
 tweet = re.sub(r"how's", "how is", tweet)
 tweet = re.sub(r"How's", "How is", tweet)
 tweet = re.sub(r"how're", "how are", tweet)
 tweet = re.sub(r"How're", "How are", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"Let's", "Let us", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"Haven't", "Have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"ain't", "am not", tweet)
 tweet = re.sub(r"Ain't", "am not", tweet)
 tweet = re.sub(r"don't", "do not", tweet)
 tweet = re.sub(r"Don't", "do not", tweet)
 tweet = re.sub(r"DON'T", "Do not", tweet)
 tweet = re.sub(r"didn't", "did not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"DIDN'T", "Did not", tweet)
 tweet = re.sub(r"doesn't", "does not", tweet)
 tweet = re.sub(r"can't", "cannot", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"Could've", "Could have", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 return tweet
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape<merge>
def specific_corrections(tweet):

 tweet = re.sub(r"b/c", "because", tweet)
 tweet = re.sub(r"w/e", "whatever", tweet)
 tweet = re.sub(r"w/out", "without", tweet)
 tweet = re.sub(r"w/o", "without", tweet)
 tweet = re.sub(r"w/", "with ", tweet)
 tweet = re.sub(r"<3", "love", tweet)
 tweet = re.sub(r"c/o", "care of", tweet)
 tweet = re.sub(r"p/u", "pick up", tweet)
 tweet = re.sub(r"
", " ", tweet)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"recentlu", "recently", tweet)
 tweet = re.sub(r"Ph0tos", "Photos", tweet)
 tweet = re.sub(r"exp0sed", "exposed", tweet)
 tweet = re.sub(r"amageddon", "armageddon", tweet)
 tweet = re.sub(r"TRAUMATISED", "traumatized", tweet)
 tweet = re.sub(r"Newss", "News", tweet)
 tweet = re.sub(r"remedyyyy", "remedy", tweet)
 tweet = re.sub(r"Bstrd", "bastard", tweet)
 tweet = re.sub(r"bldy", "bloody", tweet)
 tweet = re.sub(r"epicenterr", "epicenter", tweet)
 tweet = re.sub(r"approachng", "approaching", tweet)
 tweet = re.sub(r"evng", "evening", tweet)
 tweet = re.sub(r"Sumthng", "something", tweet)
 tweet = re.sub(r"kostumes", "costumes", tweet)
 tweet = re.sub(r"glowng", "glowing", tweet)
 tweet = re.sub(r"kindlng", "kindling", tweet)
 tweet = re.sub(r"riggd", "rigged", tweet)
 tweet = re.sub(r"HLPS", "helps", tweet)
 tweet = re.sub(r"SNCTIONS", "sanctions", tweet)
 tweet = re.sub(r"Politifiact", "PolitiFact", tweet)
 tweet = re.sub(r"Kowing", "Knowing", tweet)
 tweet = re.sub(r"wrld", "world", tweet)
 tweet = re.sub(r"shld", "should", tweet)
 tweet = re.sub(r"thruuu", "through", tweet)
 tweet = re.sub(r"probaly", "probably", tweet)
 tweet = re.sub(r"whatevs", "whatever", tweet)
 tweet = re.sub(r"colomr", "colour", tweet)
 tweet = re.sub(r"pileq", "pile", tweet)
 tweet = re.sub(r"firefightr", "firefighter", tweet)
 tweet = re.sub(r"LAIGHIGN", "laughing", tweet)
 tweet = re.sub(r"EXCLUSIV", "Exclusive", tweet)
 tweet = re.sub(r"belo-ooow", "below", tweet)
 tweet = re.sub(r"who-ooo-ole", "whole", tweet)
 tweet = re.sub(r"brother-n-law", "father-in-law", tweet)
 tweet = re.sub(r"referencereference", "reference", tweet)
 tweet = re.sub(r"IranDeal", "Iran Deal", tweet)
 tweet = re.sub(r"ProphetMuhammad", "Prophet Muhammad", tweet)
 tweet = re.sub(r"StrategicPatience", "Strategic Patience", tweet)
 tweet = re.sub(r"NASAHurricane", "NASA Hurricane", tweet)
 tweet = re.sub(r"onlinecommunities", "online communities", tweet)
 tweet = re.sub(r"LakeCounty", "Lake County", tweet)
 tweet = re.sub(r"thankU", "thank you", tweet)
 tweet = re.sub(r"iTunesMusic", "iTunes Music", tweet)
 tweet = re.sub(r"OffensiveContent", "Offensive Content", tweet)
 tweet = re.sub(r"WorstSummerJob", "Worst Summer Job", tweet)
 tweet = re.sub(r"NASASolarSystem", "NASA Solar System", tweet)
 tweet = re.sub(r"animalrescue", "animal rescue", tweet)
 tweet = re.sub(r"Ptbo", "Peterborough", tweet)
 tweet = re.sub(r"Throwingknifes", "Throwing knives", tweet)
 tweet = re.sub(r"NestleIndia", "Nestle India", tweet)
 tweet = re.sub(r"weathernetwork", "weather network", tweet)
 tweet = re.sub(r"GOPDebate", "GOP Debate", tweet)
 tweet = re.sub(r"volcanoinRussia", "volcano in Russia", tweet)
 tweet = re.sub(r"53inch", "53 inch", tweet)
 tweet = re.sub(r"FaroeIslands", "Faroe Islands", tweet)
 tweet = re.sub(r"UTC2015", "UTC 2015", tweet)
 tweet = re.sub(r"Time2015", "Time 2015", tweet)
 tweet = re.sub(r"LivingSafely", "Living Safely", tweet)
 tweet = re.sub(r"FIFA16", "Fifa 2016", tweet)
 tweet = re.sub(r"bbcnews", "bbc news", tweet)
 tweet = re.sub(r"UndergroundRailraod", "Underground Railraod", tweet)
 tweet = re.sub(r"NoSurrender", "No Surrender", tweet)
 tweet = re.sub(r"greatbritishbakeoff", "great british bake off", tweet)
 tweet = re.sub(r"LondonFire", "London Fire", tweet)
 tweet = re.sub(r"KOTAWeather", "KOTA Weather", tweet)
 tweet = re.sub(r"LuchaUnderground", "Lucha Underground", tweet)
 tweet = re.sub(r"KOIN6News", "KOIN 6 News", tweet)
 tweet = re.sub(r"9NewsGoldCoast", "9 News Gold Coast", tweet)
 tweet = re.sub(r"BlackLivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"ENGvAUS", "England vs Australia", tweet)
 tweet = re.sub(r"PlannedParenthood", "Planned Parenthood", tweet)
 tweet = re.sub(r"calgaryweather", "Calgary Weather", tweet)
 tweet = re.sub(r"renew911health", "renew 911 health", tweet)
 tweet = re.sub(r"pdx911", "Portland Police", tweet)
 tweet = re.sub(r"NJTurnpike", "New Jersey Turnpike", tweet)
 tweet = re.sub(r"HannaPH", "Typhoon Hanna", tweet)
 tweet = re.sub(r"cnnbrk", "CNN Breaking News", tweet)
 tweet = re.sub(r"IndianNews", "Indian News", tweet)
 tweet = re.sub(r"Daesh", "ISIS", tweet)
 tweet = re.sub(r"FoxNew", "Fox News", tweet)
 tweet = re.sub(r"RohnertParkDPS", "Rohnert Park DPS", tweet)
 tweet = re.sub(r"FantasticFour", "Fantastic Four", tweet)
 tweet = re.sub(r"BathAndNorthEastSomerset", "Bath and North East Somerset", tweet)
 tweet = re.sub(r"residualincome", "residual income", tweet)
 tweet = re.sub(r"YahooNewsDigest", "Yahoo News Digest", tweet)
 tweet = re.sub(r"MalaysiaAirlines", "Malaysia Airlines", tweet)
 tweet = re.sub(r"AmazonDeals", "Amazon Deals", tweet)
 tweet = re.sub(r"EndConflict", "End Conflict", tweet)
 tweet = re.sub(r"EndOccupation", "End Occupation", tweet)
 tweet = re.sub(r"KindleCountdown", "Kindle Countdown", tweet)
 tweet = re.sub(r"NoMoreHandouts", "No More Handouts", tweet)
 tweet = re.sub(r"WindstormInsurer", "Windstorm Insurer", tweet)
 tweet = re.sub(r"USAgov", "USA government", tweet)
 tweet = re.sub(r"US govt", "USA government", tweet)
 tweet = re.sub(r"WAwildfire", "WA Wildfire", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"newnewnew", "new new new", tweet)
 tweet = re.sub(r"freshoutofthebox", "fresh out of the box", tweet)
 tweet = re.sub(r"yycweather", "Calgary Weather", tweet)
 tweet = re.sub(r"calgarysun", "Calgary Sun", tweet)
 tweet = re.sub(r"shondarhimes", "Shonda Rhimes", tweet)
 tweet = re.sub(r"SushmaSwaraj", "Sushma Swaraj", tweet)
 tweet = re.sub(r"pray4japan", "Pray for Japan", tweet)
 tweet = re.sub(r"hope4japan", "Hope for Japan", tweet)
 tweet = re.sub(r"Illusionimagess", "Illusion images", tweet)
 tweet = re.sub(r"ShallWeDance", "Shall We Dance", tweet)
 tweet = re.sub(r"TCMParty", "TCM Party", tweet)
 tweet = re.sub(r"marijuananews", "marijuana news", tweet)
 tweet = re.sub(r"HeadlinesApp", "Headlines App", tweet)
 tweet = re.sub(r"BBCNewsAsia", "BBC News Asia", tweet)
 tweet = re.sub(r"BombEffects", "Bomb Effects", tweet)
 tweet = re.sub(r"idkidk", "idk idk", tweet)
 tweet = re.sub(r"BBCLive", "BBC Live", tweet)
 tweet = re.sub(r"NaturalBirth", "Natural Birth", tweet)
 tweet = re.sub(r"FusionFestival", "Fusion Festival", tweet)
 tweet = re.sub(r"50Mixed", "50 Mixed", tweet)
 tweet = re.sub(r"NoAgenda", "No Agenda", tweet)
 tweet = re.sub(r"WhiteGenocide", "White Genocide", tweet)
 tweet = re.sub(r"dirtylying", "dirty lying", tweet)
 tweet = re.sub(r"SyrianRefugees", "Syrian Refugees", tweet)
 tweet = re.sub(r"Auspol", "Australia Politics", tweet)
 tweet = re.sub(r"WhiteTerrorism", "White Terrorism", tweet)
 tweet = re.sub(r"truthfrequencyradio", "Truth Frequency Radio", tweet)
 tweet = re.sub(r"ErasureIsNotEquality", "Erasure is not equality", tweet)
 tweet = re.sub(r"toopainful", "too painful", tweet)
 tweet = re.sub(r"melindahaunton", "Melinda Haunton", tweet)
 tweet = re.sub(r"NoNukes", "No Nukes", tweet)
 tweet = re.sub(r"curryspcworld", "Currys PC World", tweet)
 tweet = re.sub(r"blackforestgateau", "black forest gateau", tweet)
 tweet = re.sub(r"BBCOne", "BBC One", tweet)
 tweet = re.sub(r"sebastianstanisaliveandwell", "Sebastian Stan is alive and well", tweet)
 tweet = re.sub(r"concertphotography", "concert photography", tweet)
 tweet = re.sub(r"TheaterTrial", "Theater Trial", tweet)
 tweet = re.sub(r"TheBrooklynLife", "The Brooklyn Life", tweet)
 tweet = re.sub(r"jokethey", "joke they", tweet)
 tweet = re.sub(r"nflweek1picks", "NFL week 1 picks", tweet)
 tweet = re.sub(r"nflnetwork", "NFL Network", tweet)
 tweet = re.sub(r"NYDNSports", "NY Daily News Sports", tweet)
 tweet = re.sub(r"crunchysensible", "crunchy sensible", tweet)
 tweet = re.sub(r"RandomActsOfRomance", "Random acts of romance", tweet)
 tweet = re.sub(r"MomentsAtHill", "Moments at hill", tweet)
 tweet = re.sub(r"liveleakfun", "live leak fun", tweet)
 tweet = re.sub(r"SahelNews", "Sahel News", tweet)
 tweet = re.sub(r"abc7newsbayarea", "ABC 7 News Bay Area", tweet)
 tweet = re.sub(r"CampLogistics", "Camp logistics", tweet)
 tweet = re.sub(r"alaskapublic", "Alaska public", tweet)
 tweet = re.sub(r"MarketResearch", "Market Research", tweet)
 tweet = re.sub(r"AccuracyEsports", "Accuracy Esports", tweet)
 tweet = re.sub(r"yychail", "Calgary hail", tweet)
 tweet = re.sub(r"yyctraffic", "Calgary traffic", tweet)
 tweet = re.sub(r"eliotschool", "eliot school", tweet)
 tweet = re.sub(r"TheBrokenCity", "The Broken City", tweet)
 tweet = re.sub(r"fieldworksmells", "field work smells", tweet)
 tweet = re.sub(r"IranElection", "Iran Election", tweet)
 tweet = re.sub(r"MyanmarFlood", "Myanmar Flood", tweet)
 tweet = re.sub(r"abc7chicago", "ABC 7 Chicago", tweet)
 tweet = re.sub(r"copolitics", "Colorado Politics", tweet)
 tweet = re.sub(r"massiveflooding", "massive flooding", tweet)
 tweet = re.sub(r"greektheatrela", "Greek Theatre Los Angeles", tweet)
 tweet = re.sub(r"publicsafetyfirst", "public safety first", tweet)
 tweet = re.sub(r"myhometown", "my hometown", tweet)
 tweet = re.sub(r"tankerfire", "tanker fire", tweet)
 tweet = re.sub(r"MEMORIALDAY", "memorial day", tweet)
 tweet = re.sub(r"MEMORIAL_DAY", "memorial day", tweet)
 tweet = re.sub(r"VirtualReality", "Virtual Reality", tweet)
 tweet = re.sub(r"mortalkombatx", "Mortal Kombat X", tweet)
 tweet = re.sub(r"mortalkombat", "Mortal Kombat", tweet)
 tweet = re.sub(r"ToshikazuKatayama", "Toshikazu Katayama", tweet)
 tweet = re.sub(r"ExtremeWeather", "Extreme Weather", tweet)
 tweet = re.sub(r"WereNotGruberVoters", "We are not gruber voters", tweet)
 tweet = re.sub(r"PhiladelphiaMuseu", "Philadelphia Museum", tweet)
 tweet = re.sub(r"NorthIowa", "North Iowa", tweet)
 tweet = re.sub(r"WillowFire", "Willow Fire", tweet)
 tweet = re.sub(r"P_EOPLE", "PEOPLE", tweet)
 tweet = re.sub(r"ThisIsAfrica", "This is Africa", tweet)
 tweet = re.sub(r"viaYouTube", "via YouTube", tweet)
 return tweet
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train.shape<drop_column>
def clean_others(tweet):
tweet = re.sub(r"2007he", "2007 he", tweet)
 tweet = re.sub(r"Hwy27", "Hwy 27", tweet)
 tweet = re.sub(r"jokethey", "joke they", tweet)
 tweet = re.sub(r"40%money", "40% money", tweet)
 tweet = re.sub(r"hegot", "he got", tweet)
 tweet = re.sub(r"wannabe", "wanna be", tweet)
 tweet = re.sub(r"dadwho", "dad who", tweet)
 tweet = re.sub(r"fundwhen", "fund when", tweet)
 tweet = re.sub(r"next chp", "next chapter", tweet)
 tweet = re.sub(r"UR sons", "your sons", tweet)
 tweet = re.sub(r"Yr voice ws", "Your voice was", tweet)
 tweet = re.sub(r"U're not", "You are not", tweet)
 tweet = re.sub(r"u'd win", "you had win", tweet)
 tweet = re.sub(r"Jus Kame", "Just came", tweet)
 tweet = re.sub(r"b4federal", "B-4, Federal", tweet)
 tweet = re.sub(r"ppor child", "poor child", tweet)
 tweet = re.sub(r"stand ogt", "stand out", tweet)
 tweet = re.sub(r"stand oup", "stand out", tweet)
 tweet = re.sub(r"IS claims", "ISIS claims", tweet)
 tweet = re.sub(r"2slow2report", "too slow to report", tweet)
 tweet = re.sub(r"@ft", "@Financial Times", tweet)
 tweet = re.sub(r"50ft", "50 ft", tweet)
 tweet = re.sub(r"Ft ABH Shadow", "featuring ABH Shadow", tweet)
 tweet = re.sub(r"Since1970the", "Since 1970 the", tweet)
 tweet = re.sub(r"whats cracking cuz", "what is cracking cause", tweet)
 tweet = re.sub(r"mentally ill", "mental illness", tweet)
 tweet = re.sub(r"RIPRIPRIP", "RIP RIP RIP", tweet)
 tweet = re.sub(r"RIPROSS", "RIP ROSS", tweet)
 tweet = re.sub(r"ABQ NM", "Albuquerque New Mexico", tweet)
 tweet = re.sub(r"
tweet = re.sub(r"in BC", "in British Columbia", tweet)
 tweet = re.sub(r"BC DROUGHT", "British Columbia Drought", tweet)
 tweet = re.sub(r"in OK", "in Oklahoma", tweet)
 tweet = re.sub(r"City OK", "City Oklahoma", tweet)
 tweet = re.sub(r"Hinton OK", "Hinton Oklahoma", tweet)
 tweet = re.sub(r"Guthrie OK", "Guthrie Oklahoma", tweet)
 tweet = re.sub(r"Choctaw OK", "Choctaw Oklahoma", tweet)
 tweet = re.sub(r"Oklahoma-OK", "Oklahoma City", tweet)
 tweet = re.sub(r"Oklahoma [OK]", "Oklahoma City", tweet)
 tweet = re.sub(r"JADE FL", "JADE Florida", tweet)
 tweet = re.sub(r"Jacksonville FL", "Jacksonville Florida", tweet)
 tweet = re.sub(r"Saint Petersburg FL", "Saint Petersburg Florida", tweet)
 tweet = re.sub(r"Wahpeton ND", "Wahpeton, North Dakota", tweet)
 tweet = re.sub(r"Northern Marians", "Northern Mariana Islands", tweet)
 tweet = re.sub(r"Northern Ma", "Northern Mariana Islands", tweet)
 tweet = re.sub(r"Dr\.", "Doctor", tweet)
 tweet = re.sub(r"f\.M\.O\.P\.", "featuring Mash Out Posse", tweet)
 tweet = re.sub(r"M\.O\.P\.", "Mash Out Posse", tweet)
 tweet = re.sub(r"M\.O\.P", "Mash Out Posse", tweet)
 tweet = re.sub(r"P\.O\.P\.E\.", "Pope", tweet)
 tweet = re.sub(r"S\.O\.S\.", "SOS", tweet)
 tweet = re.sub(r"s\.o\.s\.", "SOS", tweet)
 tweet = re.sub(r"Fire Co\.", "Fire Company", tweet)
 tweet = re.sub(r"Holt and Co\.", "Holt and Company", tweet)
 tweet = re.sub(r"roofing co\.", "roofing company", tweet)
 tweet = re.sub(r"Costa Co\.", "Costa County", tweet)
 tweet = re.sub(r"York Co\.", "York County", tweet)
 tweet = re.sub(r"Fairfax Co\.", "Fairfax County", tweet)
 tweet = re.sub(r"I\.S\.I\.S\.", "ISIS", tweet)
 tweet = re.sub(r"U\.N\.", "United Nations", tweet)
 tweet = re.sub(r"U\.S\.", "United States", tweet)
 tweet = re.sub(r"U\.S", "United States", tweet)
 tweet = re.sub(r"U\.s\.", "United States", tweet)
 tweet = re.sub(r"U\.s", "United States", tweet)
 tweet = re.sub(r"U-S\.", "United States", tweet)
 tweet = re.sub(r"U\.S National", "United States National", tweet)
 tweet = re.sub(r"LANCASTER N\.H\.", "Lancaster New Hampshire", tweet)
 tweet = re.sub(r"Manchester N\.H\.", "Manchester New Hampshire", tweet)
 tweet = re.sub(r"\:33333", "smile", tweet)
 tweet = re.sub(r"\:\)\)\)\)", "smile", tweet)
 tweet = re.sub(r"\:\)\)\)", "smile", tweet)
 tweet = re.sub(r"\:\)\)", "smile", tweet)
 tweet = re.sub(r"\:-\)", "smile", tweet)
 tweet = re.sub(r"\;-\)", "smile", tweet)
 tweet = re.sub(r"3\-D", "smile", tweet)
 tweet = re.sub(r"\:O", "smile", tweet)
 tweet = re.sub(r"\:D", "smile", tweet)
 tweet = re.sub(r"\:P", "smile", tweet)
 tweet = re.sub(r"\:p", "smile", tweet)
 tweet = re.sub(r"\;\)", "smile", tweet)
 tweet = re.sub(r"\:\)", "smile", tweet)
 tweet = re.sub(r"\=\)", "smile", tweet)
 tweet = re.sub(r"\^\^", "smile", tweet)
 tweet = re.sub(r"\:-\(", "sad", tweet)
 tweet = re.sub(r"\:\(", "sad", tweet)
 tweet = re.sub(r"\=\(", "sad", tweet)
 tweet = re.sub(r"\-\_\_\-", "", tweet)
 tweet = re.sub(r"\.\_\.", "", tweet)
 tweet = re.sub(r"T\_T", "", tweet)
 return tweet
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<drop_column>
abbreviations = {
"i.e":"that is", "mofo":"mother fucker", "til":"till",
"ft.":"featuring", "mf":"mother fucker", "bout":"about",
"ft":"featuring", "mfs":"mother fucker", "nd":"and",
"feat.":"featuring", "ltd":"limited", "nvr":"never",
"feat":"featuring", "pls":"please", "ppl":"people",
"tbs":"tablespoons", "tho":"though", "fav":"favorite",
"bc":"because", "cuz":"because", "bcuz":"because",
"btwn":"between", "fwy":"Freeway", "hwy":"Highway",
"diff":"different", "appx":"approximately",
"im":"I am", "ive":"I have", "uve":"you have",
"youd":"you had", "hadnt":"had not", "isnt":"is not",
"dont":"do not", "didnt":"did not", "cant":"cannot",
"urself":"yourself", "wont":"would not",
"heres":"Here is", "lets":"Let us", "2day":"today",
"s2g":"swear to god", "be4":"before", "b4":"before",
"4the":"for the", "1st":"first",
"okwx":"Oklahoma Weather", "arwx":"Arkansas Weather",
"gawx":"Georgia Weather", "cawx":"California Weather",
"tnwx":"Tennessee Weather", "azwx":"Arizona Weather",
"alwx":"Alabama Weather", "scwx":"South Carolina Weather",
"isis":"Islamic State", "okc":"Oklahoma","oun":"Oklahoma",
"isil":"Islamic State", "suruc":"Urfa", "pdx":"Portland",
"nm":"New Mexico", "newyork":"New York", "alska":"Alaska",
"nh":"New Hampshire", "nyc":"New York City",
"cnmi":"Northern Mariana Islands", "calif":"California",
"sarabia":"Saudi Arabia", "saudiarabia":"Saudi Arabia",
"mh370":"Malaysia Airlines Flight 370",
"12hr":"12 hr","16yr":"16 year", "hrs":"hour","hr":"hour",
"19yrs":"19 year", "yrs":"year", "min":"minute",
"20yrs":"20 year", "yr":"year", "mins":"minute",
"tren":"trend", "kno":"know", "swea":"swear", "stil":"still",
"fab":"fabulous", "srsly":"seriously", "epicente":"epicenter",
"jumpin":"jumping", "burnin":"burning", "throwin":"throwing",
"killin":"killing", "nothin":"nothing", "thinkin":"thinking",
"tryin":"trying", "lookg":"looking", "fforecast":"Forecast",
"comin":"Coming", "newss":"news", "memez":"meme", "oli":"oil",
}
def convert_abbrev(word):
return abbreviations[word.lower() ] if word.lower() in abbreviations.keys() else word
def convert_abbrev_in_text(text):
tokens = word_tokenize(text)
 tokens = [convert_abbrev(word)for word in tokens]
text = ' '.join(tokens)
 return text
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<split>
def remove_non_alnum(tweet):
punctuation = re.compile('[^A-Za-z0-9]+')
 return punctuation.sub(r' ',tweet)
 def remove_punct(text):
table = str.maketrans('', '', string.punctuation)
 return text.translate(table)
 def remove_extra_spaces(text):
text = re.sub('\s+', ' ', text ).strip()
return text
def set_train_folds(train, target, fold_number_col_name="kfold", n_splits=7, shuffle=True, random_state=42):
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=n_splits,
shuffle=shuffle,
random_state=random_state)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, fold_number_col_name] = int(f)
 folds[fold_number_col_name] = folds[fold_number_col_name].astype(int)
 return folds<categorify>
preprocessed_tweets_train = []
for tweet in tqdm_notebook(train_df['text'].values):
tweet = remove_url(tweet)
 tweet = remove_html(tweet)
 tweet = remove_emoji(tweet)
 tweet = special_characters(tweet)
 tweet = remove_nonASCII(tweet)
 tweet = expand_contractions(tweet)
 tweet = specific_corrections(tweet)
 tweet = remove_html(tweet)
 tweet = clean_others(tweet)
 tweet = convert_abbrev_in_text(tweet)
 tweet = remove_punct(tweet)
 tweet = remove_non_alnum(tweet)
 tweet = remove_extra_spaces(tweet)
 preprocessed_tweets_train.append(tweet.strip())
 train_df['text'] = preprocessed_tweets_train
folds = set_train_folds(train, target)
 folds.head()<categorify>
preprocessed_tweets_test = []
for tweet in tqdm_notebook(test_df['text'].values):
tweet = remove_url(tweet)
 tweet = remove_html(tweet)
 tweet = remove_emoji(tweet)
 tweet = special_characters(tweet)
 tweet = remove_nonASCII(tweet)
 tweet = expand_contractions(tweet)
 tweet = specific_corrections(tweet)
 tweet = remove_html(tweet)
 tweet = clean_others(tweet)
 tweet = convert_abbrev_in_text(tweet)
 tweet = remove_punct(tweet)
 tweet = remove_non_alnum(tweet)
 tweet = remove_extra_spaces(tweet)
 preprocessed_tweets_test.append(tweet.strip())
 test_df['text'] = preprocessed_tweets_test
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct 
<train_model>
df_mislabeled = train_df.groupby(['text'] ).nunique().sort_values(by='target', ascending=False)
 df_mislabeled = df_mislabeled[df_mislabeled['target'] > 1]['target']
df_mislabeled_all = df_mislabeled.index.tolist()
print(f'Number of repeated tweets(after preprocessing): {len(df_mislabeled_all)}')
 df_mislabeled_all
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0.
 for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0.
 valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<normalization>
train_df['target_new'] = train_df['target'].copy()
target_1_list = [
"POTUS Strategic Patience is a strategy for Genocide refugees IDP Internally displaced people horror etc",
"CLEARED incident with injury I 495 inner loop Exit 31 MD 97 Georgia Ave Silver Spring",
"RT NotExplained The only known image of infamous hijacker D B Cooper",
"wowo 12000 Nigerian refugees repatriated from Cameroon",
"Bayelsa poll Tension in Bayelsa as Patience Jonathan plans to hijack APC PDP Plans by former First Lady and",
"hot C 130 specially modified to land in a stadium and rescue hostages in Iran in 1980 prebreak best",
"world FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps",
"FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps",
"FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps via usatoday",
"Governor weighs parole for California school bus hijacker",
"Kosciusko police investigating pedestrian fatality hit by a train Thursday",
"A look at state actions a year after Ferguson s upheaval",
"Here is how media in Pakistan covered the capture of terrorist Mohammed Naved" ]
for mislabeled_sample in df_mislabeled_all:
if mislabeled_sample in target_1_list:
train_df.loc[train_df['text'] == mislabeled_sample, 'target_new'] = 1
else:
train_df.loc[train_df['text'] == mislabeled_sample, 'target_new'] = 0
filter_mislabel =(train_df['target'] != train_df['target_new'])
 print(f'Number of relabeled: {len(train_df[filter_mislabel])}')
 train_df[filter_mislabel][:20]
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<choose_model_class>
stop_words = set(stopwords.words("english"))
def remove_stopwords(tweet):
sentance = ' '.join(e.lower() for e in tweet.split() if e.lower() not in stop_words)
 return sentance
stemmer = SnowballStemmer("english")
 def stemming(text):
text = [stemmer.stem(word)for word in text.split() ]
return " ".join(text)
 wn = nltk.WordNetLemmatizer()
def lemmatizing(text):
text = [wn.lemmatize(word.lower())for word in text.split() ]
return " ".join(text)
 train_df['text_pure'] = train_df['text'].apply(lambda x: remove_stopwords(x))
train_df['text_pure'] = train_df['text_pure'].apply(lambda x: stemming(x))
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2619422201258426)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2619422201258426)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))<categorify>
df_mislabeled = train_df.groupby(['text_pure'] ).nunique().sort_values(by='target_new', ascending=False)
 df_mislabeled = df_mislabeled[df_mislabeled['target_new'] > 1]['target_new']
df_mislabeled_all = df_mislabeled.index.tolist()
print(f'Number of repeated tweets(after Stemming): {len(df_mislabeled_all)}')
 df_mislabeled_all
def encode_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data<define_variables>
train_df.loc[train_df['text_pure'] == "look state action year ferguson upheav", 'target_new'] = 1
df_mislabeled = train_df[train_df['text_pure'].isin(df_mislabeled_all)]
filter_mislabel =(df_mislabeled['target'] != df_mislabeled['target_new'])
 print(f'Number of relabeled: {len(df_mislabeled[filter_mislabel])}')
 df_mislabeled
feature_cols = [c for c in encode_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<define_variables>
train_df = train_df.drop('text_pure', axis=1 )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1500 
<prepare_x_and_y>
ids_target1_error = [
328,443,513,791,794,882,883,886,890,893,894,896,923,926,928,1688,1709,2033,
2063,2619,2885,3097,3640,3802,3837,3842,3900,4026,4342,4530,4533,4575,4773,
4778,4790,5223,5781,6552,6554,6570,6701,6702,6729,6731,6745,6861,6945,6965,
7201,7226,7231,7264,7494,7797,8309,8317,8329,8330,8905,8908,8913,8916,8926,
8934,8939,8972,9337,9446,9775,9791,9808,10127,10543,10552 ]
print(f'Number of ids with target1 error: {len(ids_target1_error)}' )
def run_training(fold, seed):
seed_everything(seed)
 train = encode_data(folds)
 test_ = encode_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions 
<categorify>
if ids_error_corrected:
train_df.at[train_df['id'].isin(ids_target1_error),'target_new'] = 0
train_df[train_df['id'].isin(ids_target1_error)][:20]
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<define_variables>
if ids_error_corrected:
train_df.at[train_df['id'].isin(ids_target0_error),'target_new'] = 1
train_df[train_df['id'].isin(ids_target0_error)]
SEED = [21,42]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions 
<train_model>
fig, axes = plt.subplots(1, 2, figsize=(18, 8))
axes = axes.flatten()
train_df['text_lemma'] = train_df['text'].apply(lambda x: remove_stopwords(x))
train_df['text_lemma'] = train_df['text_lemma'].apply(lambda x: lemmatizing(x))
lis = [train_df[train_df['target_new'] == 0]['text_lemma'],
train_df[train_df['target_new'] == 1]['text_lemma']]
for i, j in zip(lis, axes):
new = i.str.split()
new = new.values.tolist()
corpus = [word for i in new for word in i]
counter = Counter(corpus)
 most = counter.most_common()
x, y = [], []
for word, count in most[:30]:
if(word not in stop_words):
x.append(word)
 y.append(count)
 sns.barplot(x=y, y=x, palette='plasma', ax=j)
 axes[0].set_title('Non Disaster Tweets')
 axes[1].set_title('Disaster Tweets')
 axes[0].set_xlabel('Count')
 axes[0].set_ylabel('Word')
 axes[1].set_xlabel('Count')
 axes[1].set_ylabel('Word')
 fig.suptitle('Most Common Unigrams', fontsize=24, va='baseline')
 plt.tight_layout()
predictions2 = clfr.predict(knn_train_features)
 final_model = RandomForestClassifier(3)
 final_model.fit(np.hstack([train[target_cols], predictions2]), train_targets_scored[target_cols][TRAIN_MASK] )<merge>
def ngrams(n, title):
fig, axes = plt.subplots(1, 2, figsize=(18, 8))
axes = axes.flatten()
for i, j in zip(lis, axes):
new = i.str.split()
new = new.values.tolist()
corpus = [word for i in new for word in i]
def _get_top_ngram(corpus, n=None):
vec = CountVectorizer(ngram_range=(n, n), max_df=0.9,
stop_words='english' ).fit(corpus)
 bag_of_words = vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])
 for word, idx in vec.vocabulary_.items() ]
words_freq = sorted(words_freq, key=lambda x: x[1], reverse=True)
 return words_freq[:15]
top_n_bigrams = _get_top_ngram(i, n)[:15]
x, y = map(list, zip(*top_n_bigrams))
sns.barplot(x=y, y=x, palette='plasma', ax=j)
 axes[0].set_title('Non Disaster Tweets')
 axes[1].set_title('Disaster Tweets')
 axes[0].set_xlabel('Count')
 axes[0].set_ylabel('Words')
 axes[1].set_xlabel('Count')
 axes[1].set_ylabel('Words')
 fig.suptitle(title, fontsize=24, va='baseline')
 plt.tight_layout()
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score)
<compute_test_metric>
ngrams(2, 'Most Common Bigrams' )
def log_loss_metric(y_true, y_pred):
metrics = []
for _target in train_targets_scored.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels = [0,1]))
return np.mean(metrics )<save_to_csv>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
predictions_knn = clfr.predict(knn_test_features)
 test[target_cols] = final_model.predict(np.hstack([test[target_cols], predictions_knn]))
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<set_options>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
warnings.filterwarnings('ignore' )<import_modules>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
if Dropout_num == 0:
out = Dense(1, activation='sigmoid' )(clf_output)
 else:
x = Dropout(Dropout_num )(clf_output)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy'])
 return model
from sklearn.preprocessing import QuantileTransformer<load_from_csv>
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
train_input = bert_encode(train_df['text'].values, tokenizer, max_len=Max_length)
 test_input = bert_encode(test_df['text'].values, tokenizer, max_len=Max_length)
 train_labels = train_df['target_new'].values
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<normalization>
model = build_model(bert_layer, max_len=Max_length)
 model.summary()
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
checkpoint = ModelCheckpoint('model_BERT.h5', monitor='val_loss', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split = valid,
epochs = epochs_num,
callbacks=[checkpoint],
batch_size = batch_size_num
)
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<concatenate>
model.load_weights('model_BERT.h5')
 test_pred = model.predict(test_input)
 test_pred_int = test_pred.round().astype('int')
 train_pred = model.predict(train_input)
 train_pred_int = train_pred.round().astype('int' )
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
plt.style.use('ggplot')
 pred = pd.DataFrame(test_pred, columns=['preds'])
 pred.plot.hist()
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
submission['target'] = test_pred_int
submission.to_csv("submission_final.csv", index=False)
 submission.head(5 )
var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape 
<define_search_model>
def encode(text, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
text = tokenizer.tokenize(str(text))
text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
def fe_cluster(train, test, n_clusters_g = 35, n_clusters_c = 5, SEED = 123):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features ,test_features=fe_cluster(train_features,test_features )<feature_engineering>
ResultDict={1:'Disaster', 0:'Non Disaster'}
def predict_review(input_text):
input_seq = encode(input_text, tokenizer, max_len=Max_length)
 predict_result = model.predict(input_seq)
 i = predict_result[0][0].round().astype('int')
 print('Input:', input_text)
 pre_score = round(float(predict_result[0][0])*100, 4)
 print(f'Output: {ResultDict[i]}({pre_score}%)
 ' )
def fe_stats(train, test):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 return train, test
train_features,test_features=fe_stats(train_features,test_features )<merge>
userInput1 =
userInput2 = How did I know as soon as I walked out of class that Calgary would flood again today for sixth year in a row premium costs for windstorm insurance to climb.this time by 5 percent Truth...
 userInput6 =
userInput7 = Who is bringing the tornadoes and floods.Who is bringing the climate change.
 userInput9 =
userInput10 =
userInput11 =
userInput12 =
userInput13 =
userInput14 =
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]<drop_column>
predict_review(userInput1)
 predict_review(userInput2)
 predict_review(userInput3)
 predict_review(userInput4)
 predict_review(userInput5)
 predict_review(userInput6)
 predict_review(userInput7 )
<drop_column><EOS> 
predict_review(userInput8)
 predict_review(userInput9)
 predict_review(userInput10)
 predict_review(userInput11)
 predict_review(userInput12)
 predict_review(userInput13)
 predict_review(userInput14 )