Kaggle competition: Kannada MNIST
submission = pd.DataFrame(data={'id': test_df['id'].values, 'target': test_pred.reshape(-1,)})
 submission.head()
submission.to_csv(r'submission.csv', index=False)
 model_score = pd.DataFrame(data={'id': test_df['id'].values, 'target': test_pred.reshape(-1,), 'score': test_output.reshape(-1,)})
 model_score.to_csv(r'model_score_bert.csv', index=False)
 m.save('m_bert.h5' )<import_modules>
sns.set(rc={'figure.figsize':(11.7,5.27)})
nltk.download('stopwords')
<set_options>
df_train_raw = pd.read_csv(os.path.join('/kaggle/input/Kannada-MNIST/train.csv'))
df_valid_raw = pd.read_csv(os.path.join('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv'))
df_test_raw = pd.read_csv(os.path.join('/kaggle/input/Kannada-MNIST/test.csv'))
warnings.filterwarnings('ignore')
 stop_words = set(stopwords.words("english"))
stemmer_snowball = SnowballStemmer("english")
 stemmer_porter = PorterStemmer()
plt.style.use('ggplot')
 sns.set(font_scale=1.5)
 pd.options.display.max_columns = 250
pd.options.display.max_rows = 250<init_hyperparams>
df_train_raw=pd.concat([df_train_raw,df_valid_raw])
 df_train_raw.shape
Max_length = 42
Dropout_num = 0
learning_rate = 6e-6
valid = 0.2
epochs_num = 3
batch_size_num = 16
ids_error_corrected = True<load_from_csv>
image_x=28
image_y=28
train_path = "/kaggle/input/nlp-getting-started/train.csv"
test_path = "/kaggle/input/nlp-getting-started/test.csv"
train_df = pd.read_csv(train_path, dtype={'id': np.int16, 'target': np.int8})
 test_df = pd.read_csv(test_path, dtype={'id': np.int16})
 submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 print(f'train: {train_df.shape}')
 print(f'test: {test_df.shape}' )<count_missing_values>
x = x.astype('float32')
 x/=255.0
train_df.isnull().sum()<drop_column>
df_train_y=to_categorical(df_train_y)
 df_train_y.shape
train_df = train_df.drop(['location', 'keyword'], axis=1)
 test_df = test_df.drop(['location', 'keyword'], axis=1 )<categorify>
initial_learning_rate=0.001
def exp_decay() :
lr_schedule = ExponentialDecay(
 initial_learning_rate,
decay_steps=100000,
decay_rate=0.96,
staircase=True)
 return lr_schedule
def remove_url(tweet):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',tweet)
 def remove_html(tweet):
html=re.compile(r'<.*?>')
 return html.sub(r'',tweet)
 def remove_emoji(tweet):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', tweet )<categorify>
def early_stopping() :
return EarlyStopping(monitor="val_loss",patience=10,verbose=1 )
def special_characters(tweet):
tweet = re.sub(r"SuruÌ¤", "Suruc", tweet)
 tweet = re.sub(r"JapÌ_n", "Japan", tweet)
 tweet = re.sub(r"\x89ÛÏWhen", "When", tweet)
 tweet = re.sub(r"å£3million", "3 million", tweet)
 tweet = re.sub(r"fromåÊwounds", "from wounds", tweet)
 tweet = re.sub(r"mÌ¼sica", "music", tweet)
 tweet = re.sub(r"donå«t", "do not", tweet)
 tweet = re.sub(r"didn`t", "did not", tweet)
 tweet = re.sub(r"i\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"i\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"i\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"let\x89Ûªs", "let us", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"that\x89Ûªs", "that is", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"here\x89Ûªs", "here is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"you\x89Ûªre", "you are", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"You\x89Ûªve", "You have", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"You\x89Ûªll", "You will", tweet)
 tweet = re.sub(r"China\x89Ûªs", "China's", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"\x89Û_", "", tweet)
 tweet = re.sub(r"\x89Û¢", "", tweet)
 tweet = re.sub(r"\x89ÛÒ", "", tweet)
 tweet = re.sub(r"\x89ÛÓ", "", tweet)
 tweet = re.sub(r"\x89ÛÏ", "", tweet)
 tweet = re.sub(r"\x89Û÷", "", tweet)
 tweet = re.sub(r"\x89Ûª", "", tweet)
 tweet = re.sub(r"\x89Û¢åÊ", "", tweet)
 tweet = re.sub(r"\x89Û\x9d", "", tweet)
 tweet = re.sub(r"å_", "", tweet)
 tweet = re.sub(r"å¨", "", tweet)
 tweet = re.sub(r"åÀ", "", tweet)
 tweet = re.sub(r"åÇ", "", tweet)
 tweet = re.sub(r"åÊ", "", tweet)
 tweet = re.sub(r"åÈ", "", tweet)
 tweet = re.sub(r"Ì©", "", tweet)
 tweet = re.sub(r"&lt;", "<", tweet)
 tweet = re.sub(r"&gt;", ">", tweet)
 tweet = re.sub(r"&amp;", "&", tweet)
 return tweet
def remove_nonASCII(tweet):
tweet = ''.join([x for x in tweet if x in string.printable])
 return tweet<categorify>
def reduce_lr_platue() :
return ReduceLROnPlateau(
 monitor='val_loss', factor=0.1, patience=5, verbose=1,
)
def expand_contractions(tweet):
tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"i'M", "I am", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"I'd", "I would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"I'll", "I will", tweet)
 tweet = re.sub(r"i've", "I have", tweet)
 tweet = re.sub(r"I've", "I have", tweet)
 tweet = re.sub(r"you're", "you are", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"you'd", "you would", tweet)
 tweet = re.sub(r"You'd", "You would", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"You've", "You have", tweet)
 tweet = re.sub(r"you'll", "you will", tweet)
 tweet = re.sub(r"You'll", "You will", tweet)
 tweet = re.sub(r"y'know", "you know", tweet)
 tweet = re.sub(r"Y'know", "You know", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"We'd", "We would", tweet)
 tweet = re.sub(r"WE'VE", "We have", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"We'll", "We will", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"They'd", "They would", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"They've", "They have", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"They'll", "They will", tweet)
 tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"He'll", "He will", tweet)
 tweet = re.sub(r"she's", "she is", tweet)
 tweet = re.sub(r"She's", "She is", tweet)
 tweet = re.sub(r"she'll", "she will", tweet)
 tweet = re.sub(r"She'll", "She will", tweet)
 tweet = re.sub(r"it's", "it is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"It'll", "It will", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"Isn't", "Is not", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"Who's", "Who is", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"that's", "that is", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"here's", "here is", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Where's", "Where is", tweet)
 tweet = re.sub(r"wHeRE's", "where is", tweet)
 tweet = re.sub(r"how's", "how is", tweet)
 tweet = re.sub(r"How's", "How is", tweet)
 tweet = re.sub(r"how're", "how are", tweet)
 tweet = re.sub(r"How're", "How are", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"Let's", "Let us", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"Haven't", "Have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"ain't", "am not", tweet)
 tweet = re.sub(r"Ain't", "am not", tweet)
 tweet = re.sub(r"don't", "do not", tweet)
 tweet = re.sub(r"Don't", "do not", tweet)
 tweet = re.sub(r"DON'T", "Do not", tweet)
 tweet = re.sub(r"didn't", "did not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"DIDN'T", "Did not", tweet)
 tweet = re.sub(r"doesn't", "does not", tweet)
 tweet = re.sub(r"can't", "cannot", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"Could've", "Could have", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 return tweet<categorify>
seed=7
np.random.seed(seed)
 X_train, X_test, y_train, y_test = train_test_split(x, df_train_y, test_size=0.10, random_state=seed, shuffle=True )
def specific_corrections(tweet):

 tweet = re.sub(r"b/c", "because", tweet)
 tweet = re.sub(r"w/e", "whatever", tweet)
 tweet = re.sub(r"w/out", "without", tweet)
 tweet = re.sub(r"w/o", "without", tweet)
 tweet = re.sub(r"w/", "with ", tweet)
 tweet = re.sub(r"<3", "love", tweet)
 tweet = re.sub(r"c/o", "care of", tweet)
 tweet = re.sub(r"p/u", "pick up", tweet)
 tweet = re.sub(r"
", " ", tweet)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"recentlu", "recently", tweet)
 tweet = re.sub(r"Ph0tos", "Photos", tweet)
 tweet = re.sub(r"exp0sed", "exposed", tweet)
 tweet = re.sub(r"amageddon", "armageddon", tweet)
 tweet = re.sub(r"TRAUMATISED", "traumatized", tweet)
 tweet = re.sub(r"Newss", "News", tweet)
 tweet = re.sub(r"remedyyyy", "remedy", tweet)
 tweet = re.sub(r"Bstrd", "bastard", tweet)
 tweet = re.sub(r"bldy", "bloody", tweet)
 tweet = re.sub(r"epicenterr", "epicenter", tweet)
 tweet = re.sub(r"approachng", "approaching", tweet)
 tweet = re.sub(r"evng", "evening", tweet)
 tweet = re.sub(r"Sumthng", "something", tweet)
 tweet = re.sub(r"kostumes", "costumes", tweet)
 tweet = re.sub(r"glowng", "glowing", tweet)
 tweet = re.sub(r"kindlng", "kindling", tweet)
 tweet = re.sub(r"riggd", "rigged", tweet)
 tweet = re.sub(r"HLPS", "helps", tweet)
 tweet = re.sub(r"SNCTIONS", "sanctions", tweet)
 tweet = re.sub(r"Politifiact", "PolitiFact", tweet)
 tweet = re.sub(r"Kowing", "Knowing", tweet)
 tweet = re.sub(r"wrld", "world", tweet)
 tweet = re.sub(r"shld", "should", tweet)
 tweet = re.sub(r"thruuu", "through", tweet)
 tweet = re.sub(r"probaly", "probably", tweet)
 tweet = re.sub(r"whatevs", "whatever", tweet)
 tweet = re.sub(r"colomr", "colour", tweet)
 tweet = re.sub(r"pileq", "pile", tweet)
 tweet = re.sub(r"firefightr", "firefighter", tweet)
 tweet = re.sub(r"LAIGHIGN", "laughing", tweet)
 tweet = re.sub(r"EXCLUSIV", "Exclusive", tweet)
 tweet = re.sub(r"belo-ooow", "below", tweet)
 tweet = re.sub(r"who-ooo-ole", "whole", tweet)
 tweet = re.sub(r"brother-n-law", "father-in-law", tweet)
 tweet = re.sub(r"referencereference", "reference", tweet)
 tweet = re.sub(r"IranDeal", "Iran Deal", tweet)
 tweet = re.sub(r"ProphetMuhammad", "Prophet Muhammad", tweet)
 tweet = re.sub(r"StrategicPatience", "Strategic Patience", tweet)
 tweet = re.sub(r"NASAHurricane", "NASA Hurricane", tweet)
 tweet = re.sub(r"onlinecommunities", "online communities", tweet)
 tweet = re.sub(r"LakeCounty", "Lake County", tweet)
 tweet = re.sub(r"thankU", "thank you", tweet)
 tweet = re.sub(r"iTunesMusic", "iTunes Music", tweet)
 tweet = re.sub(r"OffensiveContent", "Offensive Content", tweet)
 tweet = re.sub(r"WorstSummerJob", "Worst Summer Job", tweet)
 tweet = re.sub(r"NASASolarSystem", "NASA Solar System", tweet)
 tweet = re.sub(r"animalrescue", "animal rescue", tweet)
 tweet = re.sub(r"Ptbo", "Peterborough", tweet)
 tweet = re.sub(r"Throwingknifes", "Throwing knives", tweet)
 tweet = re.sub(r"NestleIndia", "Nestle India", tweet)
 tweet = re.sub(r"weathernetwork", "weather network", tweet)
 tweet = re.sub(r"GOPDebate", "GOP Debate", tweet)
 tweet = re.sub(r"volcanoinRussia", "volcano in Russia", tweet)
 tweet = re.sub(r"53inch", "53 inch", tweet)
 tweet = re.sub(r"FaroeIslands", "Faroe Islands", tweet)
 tweet = re.sub(r"UTC2015", "UTC 2015", tweet)
 tweet = re.sub(r"Time2015", "Time 2015", tweet)
 tweet = re.sub(r"LivingSafely", "Living Safely", tweet)
 tweet = re.sub(r"FIFA16", "Fifa 2016", tweet)
 tweet = re.sub(r"bbcnews", "bbc news", tweet)
 tweet = re.sub(r"UndergroundRailraod", "Underground Railraod", tweet)
 tweet = re.sub(r"NoSurrender", "No Surrender", tweet)
 tweet = re.sub(r"greatbritishbakeoff", "great british bake off", tweet)
 tweet = re.sub(r"LondonFire", "London Fire", tweet)
 tweet = re.sub(r"KOTAWeather", "KOTA Weather", tweet)
 tweet = re.sub(r"LuchaUnderground", "Lucha Underground", tweet)
 tweet = re.sub(r"KOIN6News", "KOIN 6 News", tweet)
 tweet = re.sub(r"9NewsGoldCoast", "9 News Gold Coast", tweet)
 tweet = re.sub(r"BlackLivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"ENGvAUS", "England vs Australia", tweet)
 tweet = re.sub(r"PlannedParenthood", "Planned Parenthood", tweet)
 tweet = re.sub(r"calgaryweather", "Calgary Weather", tweet)
 tweet = re.sub(r"renew911health", "renew 911 health", tweet)
 tweet = re.sub(r"pdx911", "Portland Police", tweet)
 tweet = re.sub(r"NJTurnpike", "New Jersey Turnpike", tweet)
 tweet = re.sub(r"HannaPH", "Typhoon Hanna", tweet)
 tweet = re.sub(r"cnnbrk", "CNN Breaking News", tweet)
 tweet = re.sub(r"IndianNews", "Indian News", tweet)
 tweet = re.sub(r"Daesh", "ISIS", tweet)
 tweet = re.sub(r"FoxNew", "Fox News", tweet)
 tweet = re.sub(r"RohnertParkDPS", "Rohnert Park DPS", tweet)
 tweet = re.sub(r"FantasticFour", "Fantastic Four", tweet)
 tweet = re.sub(r"BathAndNorthEastSomerset", "Bath and North East Somerset", tweet)
 tweet = re.sub(r"residualincome", "residual income", tweet)
 tweet = re.sub(r"YahooNewsDigest", "Yahoo News Digest", tweet)
 tweet = re.sub(r"MalaysiaAirlines", "Malaysia Airlines", tweet)
 tweet = re.sub(r"AmazonDeals", "Amazon Deals", tweet)
 tweet = re.sub(r"EndConflict", "End Conflict", tweet)
 tweet = re.sub(r"EndOccupation", "End Occupation", tweet)
 tweet = re.sub(r"KindleCountdown", "Kindle Countdown", tweet)
 tweet = re.sub(r"NoMoreHandouts", "No More Handouts", tweet)
 tweet = re.sub(r"WindstormInsurer", "Windstorm Insurer", tweet)
 tweet = re.sub(r"USAgov", "USA government", tweet)
 tweet = re.sub(r"US govt", "USA government", tweet)
 tweet = re.sub(r"WAwildfire", "WA Wildfire", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"newnewnew", "new new new", tweet)
 tweet = re.sub(r"freshoutofthebox", "fresh out of the box", tweet)
 tweet = re.sub(r"yycweather", "Calgary Weather", tweet)
 tweet = re.sub(r"calgarysun", "Calgary Sun", tweet)
 tweet = re.sub(r"shondarhimes", "Shonda Rhimes", tweet)
 tweet = re.sub(r"SushmaSwaraj", "Sushma Swaraj", tweet)
 tweet = re.sub(r"pray4japan", "Pray for Japan", tweet)
 tweet = re.sub(r"hope4japan", "Hope for Japan", tweet)
 tweet = re.sub(r"Illusionimagess", "Illusion images", tweet)
 tweet = re.sub(r"ShallWeDance", "Shall We Dance", tweet)
 tweet = re.sub(r"TCMParty", "TCM Party", tweet)
 tweet = re.sub(r"marijuananews", "marijuana news", tweet)
 tweet = re.sub(r"HeadlinesApp", "Headlines App", tweet)
 tweet = re.sub(r"BBCNewsAsia", "BBC News Asia", tweet)
 tweet = re.sub(r"BombEffects", "Bomb Effects", tweet)
 tweet = re.sub(r"idkidk", "idk idk", tweet)
 tweet = re.sub(r"BBCLive", "BBC Live", tweet)
 tweet = re.sub(r"NaturalBirth", "Natural Birth", tweet)
 tweet = re.sub(r"FusionFestival", "Fusion Festival", tweet)
 tweet = re.sub(r"50Mixed", "50 Mixed", tweet)
 tweet = re.sub(r"NoAgenda", "No Agenda", tweet)
 tweet = re.sub(r"WhiteGenocide", "White Genocide", tweet)
 tweet = re.sub(r"dirtylying", "dirty lying", tweet)
 tweet = re.sub(r"SyrianRefugees", "Syrian Refugees", tweet)
 tweet = re.sub(r"Auspol", "Australia Politics", tweet)
 tweet = re.sub(r"WhiteTerrorism", "White Terrorism", tweet)
 tweet = re.sub(r"truthfrequencyradio", "Truth Frequency Radio", tweet)
 tweet = re.sub(r"ErasureIsNotEquality", "Erasure is not equality", tweet)
 tweet = re.sub(r"toopainful", "too painful", tweet)
 tweet = re.sub(r"melindahaunton", "Melinda Haunton", tweet)
 tweet = re.sub(r"NoNukes", "No Nukes", tweet)
 tweet = re.sub(r"curryspcworld", "Currys PC World", tweet)
 tweet = re.sub(r"blackforestgateau", "black forest gateau", tweet)
 tweet = re.sub(r"BBCOne", "BBC One", tweet)
 tweet = re.sub(r"sebastianstanisaliveandwell", "Sebastian Stan is alive and well", tweet)
 tweet = re.sub(r"concertphotography", "concert photography", tweet)
 tweet = re.sub(r"TheaterTrial", "Theater Trial", tweet)
 tweet = re.sub(r"TheBrooklynLife", "The Brooklyn Life", tweet)
 tweet = re.sub(r"jokethey", "joke they", tweet)
 tweet = re.sub(r"nflweek1picks", "NFL week 1 picks", tweet)
 tweet = re.sub(r"nflnetwork", "NFL Network", tweet)
 tweet = re.sub(r"NYDNSports", "NY Daily News Sports", tweet)
 tweet = re.sub(r"crunchysensible", "crunchy sensible", tweet)
 tweet = re.sub(r"RandomActsOfRomance", "Random acts of romance", tweet)
 tweet = re.sub(r"MomentsAtHill", "Moments at hill", tweet)
 tweet = re.sub(r"liveleakfun", "live leak fun", tweet)
 tweet = re.sub(r"SahelNews", "Sahel News", tweet)
 tweet = re.sub(r"abc7newsbayarea", "ABC 7 News Bay Area", tweet)
 tweet = re.sub(r"CampLogistics", "Camp logistics", tweet)
 tweet = re.sub(r"alaskapublic", "Alaska public", tweet)
 tweet = re.sub(r"MarketResearch", "Market Research", tweet)
 tweet = re.sub(r"AccuracyEsports", "Accuracy Esports", tweet)
 tweet = re.sub(r"yychail", "Calgary hail", tweet)
 tweet = re.sub(r"yyctraffic", "Calgary traffic", tweet)
 tweet = re.sub(r"eliotschool", "eliot school", tweet)
 tweet = re.sub(r"TheBrokenCity", "The Broken City", tweet)
 tweet = re.sub(r"fieldworksmells", "field work smells", tweet)
 tweet = re.sub(r"IranElection", "Iran Election", tweet)
 tweet = re.sub(r"MyanmarFlood", "Myanmar Flood", tweet)
 tweet = re.sub(r"abc7chicago", "ABC 7 Chicago", tweet)
 tweet = re.sub(r"copolitics", "Colorado Politics", tweet)
 tweet = re.sub(r"massiveflooding", "massive flooding", tweet)
 tweet = re.sub(r"greektheatrela", "Greek Theatre Los Angeles", tweet)
 tweet = re.sub(r"publicsafetyfirst", "public safety first", tweet)
 tweet = re.sub(r"myhometown", "my hometown", tweet)
 tweet = re.sub(r"tankerfire", "tanker fire", tweet)
 tweet = re.sub(r"MEMORIALDAY", "memorial day", tweet)
 tweet = re.sub(r"MEMORIAL_DAY", "memorial day", tweet)
 tweet = re.sub(r"VirtualReality", "Virtual Reality", tweet)
 tweet = re.sub(r"mortalkombatx", "Mortal Kombat X", tweet)
 tweet = re.sub(r"mortalkombat", "Mortal Kombat", tweet)
 tweet = re.sub(r"ToshikazuKatayama", "Toshikazu Katayama", tweet)
 tweet = re.sub(r"ExtremeWeather", "Extreme Weather", tweet)
 tweet = re.sub(r"WereNotGruberVoters", "We are not gruber voters", tweet)
 tweet = re.sub(r"PhiladelphiaMuseu", "Philadelphia Museum", tweet)
 tweet = re.sub(r"NorthIowa", "North Iowa", tweet)
 tweet = re.sub(r"WillowFire", "Willow Fire", tweet)
 tweet = re.sub(r"P_EOPLE", "PEOPLE", tweet)
 tweet = re.sub(r"ThisIsAfrica", "This is Africa", tweet)
 tweet = re.sub(r"viaYouTube", "via YouTube", tweet)
 return tweet<categorify>
inp_layer = Input(shape=(image_x, image_y, 1))
conv_1=Conv2D(64, kernel_size=(1, 1), padding='same', activation='relu' )(inp_layer)
 max_1=MaxPooling2D(2,2 )(conv_1)
 conv_2=Conv2D(64, kernel_size=(2, 2), padding='same', activation='relu' )(inp_layer)
 max_2=MaxPooling2D(2,2 )(conv_2)
 conv_3=Conv2D(64, kernel_size=(3, 3), padding='same', activation='relu' )(inp_layer)
 max_3=MaxPooling2D(2,2 )(conv_3)
 merged = maximum([max_1, max_2, max_3])
 bn=BatchNormalization()(merged)
 dr=Dropout(0.20 )(bn)
 conv_1=Conv2D(128, kernel_size=(1, 1), padding='same', activation='relu' )(dr)
 max_1=MaxPooling2D(2,2 )(conv_1)
 conv_2=Conv2D(128, kernel_size=(2, 2), padding='same', activation='relu' )(dr)
 max_2=MaxPooling2D(2,2 )(conv_2)
 conv_3=Conv2D(128, kernel_size=(3, 3), padding='same', activation='relu' )(dr)
 max_3=MaxPooling2D(2,2 )(conv_3)
 merged = maximum([max_1, max_2, max_3])
 bn=BatchNormalization()(merged)
 dr=Dropout(0.20 )(bn)
 model= Model(inp_layer, dr)
 model_cnn2 = Sequential()
model_cnn2.add(model)
 model_cnn2.add(Conv2D(256, kernel_size=3, activation='relu'))
model_cnn2.add(BatchNormalization())
 model_cnn2.add(Flatten())
 model_cnn2.add(Dropout(0.25))
model_cnn2.add(Dense(512,activation='relu'))
model_cnn2.add(BatchNormalization())
 model_cnn2.add(Dropout(0.20))
model_cnn2.add(Dense(64,activation='relu'))
model_cnn2.add(BatchNormalization())
 model_cnn2.add(Dropout(0.10))
model_cnn2.add(Dense(10,activation='softmax'))
model_cnn2.summary()
optimizer = RMSprop(lr=0.001)
 model_cnn2.compile(loss='categorical_crossentropy',metrics="accuracy"
, optimizer=optimizer )
def clean_others(tweet):
tweet = re.sub(r"2007he", "2007 he", tweet)
 tweet = re.sub(r"Hwy27", "Hwy 27", tweet)
 tweet = re.sub(r"jokethey", "joke they", tweet)
 tweet = re.sub(r"40%money", "40% money", tweet)
 tweet = re.sub(r"hegot", "he got", tweet)
 tweet = re.sub(r"wannabe", "wanna be", tweet)
 tweet = re.sub(r"dadwho", "dad who", tweet)
 tweet = re.sub(r"fundwhen", "fund when", tweet)
 tweet = re.sub(r"next chp", "next chapter", tweet)
 tweet = re.sub(r"UR sons", "your sons", tweet)
 tweet = re.sub(r"Yr voice ws", "Your voice was", tweet)
 tweet = re.sub(r"U're not", "You are not", tweet)
 tweet = re.sub(r"u'd win", "you had win", tweet)
 tweet = re.sub(r"Jus Kame", "Just came", tweet)
 tweet = re.sub(r"b4federal", "B-4, Federal", tweet)
 tweet = re.sub(r"ppor child", "poor child", tweet)
 tweet = re.sub(r"stand ogt", "stand out", tweet)
 tweet = re.sub(r"stand oup", "stand out", tweet)
 tweet = re.sub(r"IS claims", "ISIS claims", tweet)
 tweet = re.sub(r"2slow2report", "too slow to report", tweet)
 tweet = re.sub(r"@ft", "@Financial Times", tweet)
 tweet = re.sub(r"50ft", "50 ft", tweet)
 tweet = re.sub(r"Ft ABH Shadow", "featuring ABH Shadow", tweet)
 tweet = re.sub(r"Since1970the", "Since 1970 the", tweet)
 tweet = re.sub(r"whats cracking cuz", "what is cracking cause", tweet)
 tweet = re.sub(r"mentally ill", "mental illness", tweet)
 tweet = re.sub(r"RIPRIPRIP", "RIP RIP RIP", tweet)
 tweet = re.sub(r"RIPROSS", "RIP ROSS", tweet)
 tweet = re.sub(r"ABQ NM", "Albuquerque New Mexico", tweet)
 tweet = re.sub(r"
tweet = re.sub(r"in BC", "in British Columbia", tweet)
 tweet = re.sub(r"BC DROUGHT", "British Columbia Drought", tweet)
 tweet = re.sub(r"in OK", "in Oklahoma", tweet)
 tweet = re.sub(r"City OK", "City Oklahoma", tweet)
 tweet = re.sub(r"Hinton OK", "Hinton Oklahoma", tweet)
 tweet = re.sub(r"Guthrie OK", "Guthrie Oklahoma", tweet)
 tweet = re.sub(r"Choctaw OK", "Choctaw Oklahoma", tweet)
 tweet = re.sub(r"Oklahoma-OK", "Oklahoma City", tweet)
 tweet = re.sub(r"Oklahoma [OK]", "Oklahoma City", tweet)
 tweet = re.sub(r"JADE FL", "JADE Florida", tweet)
 tweet = re.sub(r"Jacksonville FL", "Jacksonville Florida", tweet)
 tweet = re.sub(r"Saint Petersburg FL", "Saint Petersburg Florida", tweet)
 tweet = re.sub(r"Wahpeton ND", "Wahpeton, North Dakota", tweet)
 tweet = re.sub(r"Northern Marians", "Northern Mariana Islands", tweet)
 tweet = re.sub(r"Northern Ma", "Northern Mariana Islands", tweet)
 tweet = re.sub(r"Dr\.", "Doctor", tweet)
 tweet = re.sub(r"f\.M\.O\.P\.", "featuring Mash Out Posse", tweet)
 tweet = re.sub(r"M\.O\.P\.", "Mash Out Posse", tweet)
 tweet = re.sub(r"M\.O\.P", "Mash Out Posse", tweet)
 tweet = re.sub(r"P\.O\.P\.E\.", "Pope", tweet)
 tweet = re.sub(r"S\.O\.S\.", "SOS", tweet)
 tweet = re.sub(r"s\.o\.s\.", "SOS", tweet)
 tweet = re.sub(r"Fire Co\.", "Fire Company", tweet)
 tweet = re.sub(r"Holt and Co\.", "Holt and Company", tweet)
 tweet = re.sub(r"roofing co\.", "roofing company", tweet)
 tweet = re.sub(r"Costa Co\.", "Costa County", tweet)
 tweet = re.sub(r"York Co\.", "York County", tweet)
 tweet = re.sub(r"Fairfax Co\.", "Fairfax County", tweet)
 tweet = re.sub(r"I\.S\.I\.S\.", "ISIS", tweet)
 tweet = re.sub(r"U\.N\.", "United Nations", tweet)
 tweet = re.sub(r"U\.S\.", "United States", tweet)
 tweet = re.sub(r"U\.S", "United States", tweet)
 tweet = re.sub(r"U\.s\.", "United States", tweet)
 tweet = re.sub(r"U\.s", "United States", tweet)
 tweet = re.sub(r"U-S\.", "United States", tweet)
 tweet = re.sub(r"U\.S National", "United States National", tweet)
 tweet = re.sub(r"LANCASTER N\.H\.", "Lancaster New Hampshire", tweet)
 tweet = re.sub(r"Manchester N\.H\.", "Manchester New Hampshire", tweet)
 tweet = re.sub(r"\:33333", "smile", tweet)
 tweet = re.sub(r"\:\)\)\)\)", "smile", tweet)
 tweet = re.sub(r"\:\)\)\)", "smile", tweet)
 tweet = re.sub(r"\:\)\)", "smile", tweet)
 tweet = re.sub(r"\:-\)", "smile", tweet)
 tweet = re.sub(r"\;-\)", "smile", tweet)
 tweet = re.sub(r"3\-D", "smile", tweet)
 tweet = re.sub(r"\:O", "smile", tweet)
 tweet = re.sub(r"\:D", "smile", tweet)
 tweet = re.sub(r"\:P", "smile", tweet)
 tweet = re.sub(r"\:p", "smile", tweet)
 tweet = re.sub(r"\;\)", "smile", tweet)
 tweet = re.sub(r"\:\)", "smile", tweet)
 tweet = re.sub(r"\=\)", "smile", tweet)
 tweet = re.sub(r"\^\^", "smile", tweet)
 tweet = re.sub(r"\:-\(", "sad", tweet)
 tweet = re.sub(r"\:\(", "sad", tweet)
 tweet = re.sub(r"\=\(", "sad", tweet)
 tweet = re.sub(r"\-\_\_\-", "", tweet)
 tweet = re.sub(r"\.\_\.", "", tweet)
 tweet = re.sub(r"T\_T", "", tweet)
 return tweet<define_variables>
abbreviations = {
"i.e":"that is", "mofo":"mother fucker", "til":"till",
"ft.":"featuring", "mf":"mother fucker", "bout":"about",
"ft":"featuring", "mfs":"mother fucker", "nd":"and",
"feat.":"featuring", "ltd":"limited", "nvr":"never",
"feat":"featuring", "pls":"please", "ppl":"people",
"tbs":"tablespoons", "tho":"though", "fav":"favorite",
"bc":"because", "cuz":"because", "bcuz":"because",
"btwn":"between", "fwy":"Freeway", "hwy":"Highway",
"diff":"different", "appx":"approximately",
"im":"I am", "ive":"I have", "uve":"you have",
"youd":"you had", "hadnt":"had not", "isnt":"is not",
"dont":"do not", "didnt":"did not", "cant":"cannot",
"urself":"yourself", "wont":"would not",
"heres":"Here is", "lets":"Let us", "2day":"today",
"s2g":"swear to god", "be4":"before", "b4":"before",
"4the":"for the", "1st":"first",
"okwx":"Oklahoma Weather", "arwx":"Arkansas Weather",
"gawx":"Georgia Weather", "cawx":"California Weather",
"tnwx":"Tennessee Weather", "azwx":"Arizona Weather",
"alwx":"Alabama Weather", "scwx":"South Carolina Weather",
"isis":"Islamic State", "okc":"Oklahoma","oun":"Oklahoma",
"isil":"Islamic State", "suruc":"Urfa", "pdx":"Portland",
"nm":"New Mexico", "newyork":"New York", "alska":"Alaska",
"nh":"New Hampshire", "nyc":"New York City",
"cnmi":"Northern Mariana Islands", "calif":"California",
"sarabia":"Saudi Arabia", "saudiarabia":"Saudi Arabia",
"mh370":"Malaysia Airlines Flight 370",
"12hr":"12 hr","16yr":"16 year", "hrs":"hour","hr":"hour",
"19yrs":"19 year", "yrs":"year", "min":"minute",
"20yrs":"20 year", "yr":"year", "mins":"minute",
"tren":"trend", "kno":"know", "swea":"swear", "stil":"still",
"fab":"fabulous", "srsly":"seriously", "epicente":"epicenter",
"jumpin":"jumping", "burnin":"burning", "throwin":"throwing",
"killin":"killing", "nothin":"nothing", "thinkin":"thinking",
"tryin":"trying", "lookg":"looking", "fforecast":"Forecast",
"comin":"Coming", "newss":"news", "memez":"meme", "oli":"oil",
}
def convert_abbrev(word):
return abbreviations[word.lower() ] if word.lower() in abbreviations.keys() else word
def convert_abbrev_in_text(text):
tokens = word_tokenize(text)
 tokens = [convert_abbrev(word)for word in tokens]
text = ' '.join(tokens)
 return text<string_transform>
model_output_5 = model_cnn2.fit(X_train,y_train,epochs=40,batch_size=128,validation_data=(X_test,y_test),callbacks=[reduce_lr_platue() ,early_stopping() ] )
def remove_non_alnum(tweet):
punctuation = re.compile('[^A-Za-z0-9]+')
 return punctuation.sub(r' ',tweet)
 def remove_punct(text):
table = str.maketrans('', '', string.punctuation)
 return text.translate(table)
 def remove_extra_spaces(text):
text = re.sub('\s+', ' ', text ).strip()
return text<drop_column>
x_pred_total=model_cnn2.predict(X_test )
preprocessed_tweets_train = []
for tweet in tqdm_notebook(train_df['text'].values):
tweet = remove_url(tweet)
 tweet = remove_html(tweet)
 tweet = remove_emoji(tweet)
 tweet = special_characters(tweet)
 tweet = remove_nonASCII(tweet)
 tweet = expand_contractions(tweet)
 tweet = specific_corrections(tweet)
 tweet = remove_html(tweet)
 tweet = clean_others(tweet)
 tweet = convert_abbrev_in_text(tweet)
 tweet = remove_punct(tweet)
 tweet = remove_non_alnum(tweet)
 tweet = remove_extra_spaces(tweet)
 preprocessed_tweets_train.append(tweet.strip())
 train_df['text'] = preprocessed_tweets_train<drop_column>
y_list=[]
train_results=pd.DataFrame()
for i in range(len(x_pred_total)) :
y_list.append(np.argmax(x_pred_total[i]))
train_results["ypred"] = y_list
y_list=[]
for i in range(len(y_test)) :
y_list.append(np.argmax(y_test[i]))
train_results["yactual"] = y_list
preprocessed_tweets_test = []
for tweet in tqdm_notebook(test_df['text'].values):
tweet = remove_url(tweet)
 tweet = remove_html(tweet)
 tweet = remove_emoji(tweet)
 tweet = special_characters(tweet)
 tweet = remove_nonASCII(tweet)
 tweet = expand_contractions(tweet)
 tweet = specific_corrections(tweet)
 tweet = remove_html(tweet)
 tweet = clean_others(tweet)
 tweet = convert_abbrev_in_text(tweet)
 tweet = remove_punct(tweet)
 tweet = remove_non_alnum(tweet)
 tweet = remove_extra_spaces(tweet)
 preprocessed_tweets_test.append(tweet.strip())
 test_df['text'] = preprocessed_tweets_test<count_unique_values>
n_match = train_results[train_results["ypred"]!=train_results["yactual"]]
print(n_match.shape)
 n_match.groupby(by="yactual" ).count()
df_mislabeled = train_df.groupby(['text'] ).nunique().sort_values(by='target', ascending=False)
 df_mislabeled = df_mislabeled[df_mislabeled['target'] > 1]['target']
df_mislabeled_all = df_mislabeled.index.tolist()
print(f'Number of repeated tweets(after preprocessing): {len(df_mislabeled_all)}')
 df_mislabeled_all<feature_engineering>
df_test=df_test_raw.iloc[:,1:]
x = df_test.astype('float32')
 test=np.array(x ).reshape(x.shape[0],image_x,image_y,1)
 test/=255.0
train_df['target_new'] = train_df['target'].copy()
target_1_list = [
"POTUS Strategic Patience is a strategy for Genocide refugees IDP Internally displaced people horror etc",
"CLEARED incident with injury I 495 inner loop Exit 31 MD 97 Georgia Ave Silver Spring",
"RT NotExplained The only known image of infamous hijacker D B Cooper",
"wowo 12000 Nigerian refugees repatriated from Cameroon",
"Bayelsa poll Tension in Bayelsa as Patience Jonathan plans to hijack APC PDP Plans by former First Lady and",
"hot C 130 specially modified to land in a stadium and rescue hostages in Iran in 1980 prebreak best",
"world FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps",
"FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps",
"FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps via usatoday",
"Governor weighs parole for California school bus hijacker",
"Kosciusko police investigating pedestrian fatality hit by a train Thursday",
"A look at state actions a year after Ferguson s upheaval",
"Here is how media in Pakistan covered the capture of terrorist Mohammed Naved" ]
for mislabeled_sample in df_mislabeled_all:
if mislabeled_sample in target_1_list:
train_df.loc[train_df['text'] == mislabeled_sample, 'target_new'] = 1
else:
train_df.loc[train_df['text'] == mislabeled_sample, 'target_new'] = 0
filter_mislabel =(train_df['target'] != train_df['target_new'])
 print(f'Number of relabeled: {len(train_df[filter_mislabel])}')
 train_df[filter_mislabel][:20]<string_transform>
y_pred=model_cnn2.predict(test )
stop_words = set(stopwords.words("english"))
def remove_stopwords(tweet):
sentance = ' '.join(e.lower() for e in tweet.split() if e.lower() not in stop_words)
 return sentance
stemmer = SnowballStemmer("english")
 def stemming(text):
text = [stemmer.stem(word)for word in text.split() ]
return " ".join(text)
 wn = nltk.WordNetLemmatizer()
def lemmatizing(text):
text = [wn.lemmatize(word.lower())for word in text.split() ]
return " ".join(text)
 train_df['text_pure'] = train_df['text'].apply(lambda x: remove_stopwords(x))
train_df['text_pure'] = train_df['text_pure'].apply(lambda x: stemming(x))<count_unique_values>
y_list=[]
for i in range(len(y_pred)) :
y_list.append(np.argmax(y_pred[i]))
result=df_test_raw[["id"]]
result["label"] = y_list
df_mislabeled = train_df.groupby(['text_pure'] ).nunique().sort_values(by='target_new', ascending=False)
 df_mislabeled = df_mislabeled[df_mislabeled['target_new'] > 1]['target_new']
df_mislabeled_all = df_mislabeled.index.tolist()
print(f'Number of repeated tweets(after Stemming): {len(df_mislabeled_all)}')
 df_mislabeled_all<feature_engineering>
result.to_csv("submission.csv",index=False )