Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<define_variables>
warnings.filterwarnings("ignore")
 pd.options.display.max_colwidth = 170
data.show_batch(rows=4,figsize=(7,8))<choose_model_class>
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
learner = cnn_learner(data,models.resnet152, metrics=[error_rate, accuracy] )<train_model>
train.target.value_counts()
learner.fit_one_cycle(12 )<save_model>
train = train.reindex(np.random.permutation(train.index))
learner.save('kannada-stage1' )<choose_model_class>
train_len = train.shape[0]
test_len = test.shape[0]
train_keyword_null_count = train[train.keyword.isnull() == True].shape[0]
test_keyword_null_count = test[test.keyword.isnull() == True].shape[0]
train_location_null_count = train[train.location.isnull() == True].shape[0]
test_location_null_count = test[test.location.isnull() == True].shape[0]
print("Training data having {}% Keyword as Null".format(( train_keyword_null_count*100)/train_len))
print("Test data having {}% Keyword as Null
".format(( test_keyword_null_count*100)/test_len))
print("Training data having {}% Loaction as Null".format(train_location_null_count*100/train_len))
print("Test data having {}% Location as Null
".format(test_location_null_count*100/test_len))
interp = ClassificationInterpretation.from_learner(learner )<find_best_params>
train.location.isnull().value_counts()
losses,idx = interp.top_losses()<filter>
train.drop("location", axis = 1, inplace = True)
 test.drop("location", axis = 1, inplace = True )
len(data.valid_ds)==len(losses)==len(idx )<find_best_params>
train.keyword.fillna("", inplace = True)
 test.keyword.fillna("", inplace = True)
 train.text = train.text + " " + train.keyword
test.text = test.text + " " + test.keyword
learner.lr_find()<train_model>
train.drop("keyword", axis = 1, inplace = True)
 test.drop("keyword", axis = 1, inplace = True )
learner.unfreeze()
learner.fit_one_cycle(10, max_lr=slice(1e-6,1e-4))<save_model>
train_filter0 = train.target == 0
train_filter1 = train.target == 1
learner.save('kannada-stage2' )<load_pretrained>
train["length"] = train.text.map(len)
 test["length"] = test.text.map(len )
learner.load('kannada-stage2' )<define_variables>
train["word_cnt"] = train.text.apply(lambda x : len(x.split(" ")))
 test["word_cnt"] = test.text.apply(lambda x : len(x.split(" ")))
 train["a_count"] = train.text.apply(lambda x : len([char for char in str(x)if char == "@"]))
test["a_count"] = test.text.apply(lambda x : len([char for char in str(x)if char == "@"]))
train["hash_count"] = train.text.apply(lambda x : len([char for char in str(x)if char == "
test["hash_count"] = test.text.apply(lambda x : len([char for char in str(x)if char == "
img = learner.data.valid_ds[0][0]<predict_on_test>
def dict_formation(data):

 dict_word = {}
for sent in data.text.tolist() :
words = sent.split(" ")
 for word in words:
word = word.lower()
try:
dict_word[word] = dict_word[word]+1
except:
dict_word[word] = 1
return dict_word
learner.predict(img )<predict_on_test>
train_target0_words_dict = dict_formation(train[train_filter0])
 train_target1_words_dict = dict_formation(train[train_filter1])
 test_words_dict = dict_formation(test)
 train_word_dict = dict_formation(train )
<load_from_csv>
def get_ngram_dataframe(n, data, label):
train_ngram = ngrams(data.text.str.cat(sep=' ' ).split() , n=n)
 train_ngram = Counter(train_ngram)
 train_ngram = dict(train_ngram)
 train_ngram = dict(sorted(train_ngram.items() , key=lambda x: x[1], reverse=True))
train_ngram_df = pd.DataFrame()
train_ngram_df[label] = train_ngram.keys()
train_ngram_df['Count'] = train_ngram.values()
return train_ngram_df
test_csv = pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 test_csv.drop('id',axis = 'columns',inplace = True)
 sub_df = pd.DataFrame(columns=['id','label'])
 test_data = np.array(test_csv )<normalization>
%%time
glove = '.. /input/glove6b100dtxt/glove.6B.100d.txt'
print("Extracting GloVe embedding")
 embed_glove = load_embed(glove )
def get_img(data):
t1 = data.reshape(28,28)/255
t1 = np.stack([t1]*3,axis=0)
 img = Image(FloatTensor(t1))
return img<import_modules>
def build_vocab(texts):

 sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word.lower() ] += 1
except KeyError:
vocab[word.lower() ] = 1
return vocab
from fastprogress import progress_bar<feature_engineering>
def check_coverage(vocab, embeddings_index):

 known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.3%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.3%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
def decr(ido):
return ido-1
sub_df['id'] = sub_df['id'].map(decr )<save_to_csv>
vocab_train = build_vocab(train['text'])
 print("Glove : Train")
 oov_glove_train = check_coverage(vocab_train, embed_glove)
 vocab_test = build_vocab(test['text'])
 print("Glove : Test")
 oov_glove_test = check_coverage(vocab_test, embed_glove )
sub_df.to_csv("submission.csv", index=False )<load_from_csv>
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because",
"could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not",
"don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would",
"he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you",
"how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have",
"I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would",
"i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am",
"i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have",
"it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us",
"ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not",
"mightn't've": "might not have", "must've": "must have", "mustn't": "must not",
"mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have",
"o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not",
"sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have",
"she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have",
"shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as",
"this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is",
"there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is",
"they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have",
"they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would",
"we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have",
"weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is",
"what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is",
"where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have",
"why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have",
"would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all",
"y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have",
"you'd": "you would", "you'd've": "you would have",
"you'll": "you will", "you'll've": "you will have",
"you're": "you are", "you've": "you have" }
train_data = pd.read_csv(".. /input/Kannada-MNIST/train.csv")
 train_data.shape<split>
%%time
train.text = train.text.apply(lambda x: x.lower())
 test.text = test.text.apply(lambda x: x.lower() )
X_train_test = train_data.values[:, 1:]
y_train_test = train_data.label.values
X_train, X_test, y_train, y_test = train_test_split(X_train_test, y_train_test, test_size = 0.02, random_state=42)
 print('Train shapes: ', X_train.shape, y_train.shape)
 print('Test shapes: ', X_test.shape, y_test.shape )<prepare_output>
%%time
train.text = train.text.apply(lambda x : " ".join([contraction_mapping[word].lower() if word in contraction_mapping.keys() else word.lower() for word in x.split(" ")]))
test.text = test.text.apply(lambda x : " ".join([contraction_mapping[word].lower() if word in contraction_mapping.keys() else word.lower() for word in x.split(" ")]))
print(np.min(X_train), np.max(X_train))
X_train_max = np.max(X_train)
 X_train = X_train /(0.5 * X_train_max)- 1
print(np.min(X_train), np.max(X_train))
print(np.min(X_test), np.max(X_test))
X_test = X_test /(0.5 * X_train_max)- 1
print(np.min(X_test), np.max(X_test))<import_modules>
vocab_train = build_vocab(train['text'])
 print("Glove : Train")
 oov_glove_train = check_coverage(vocab_train, embed_glove)
 vocab_test = build_vocab(test['text'])
 print("Glove : Test")
 oov_glove_test = check_coverage(vocab_test, embed_glove )
from keras.layers import *
from keras.models import Sequential
from keras.optimizers import *
from keras import regularizers
from keras.utils import plot_model, model_to_dot
from IPython.display import SVG<choose_model_class>
def split_textnum(text):

 match = re.match(r"([a-z]+ )([0-9]+)", text, re.I)
 if match:
items = " ".join(list(match.groups()))
else:
match = re.match(r"([0-9]+ )([a-z]+)", text, re.I)
 if match:
items = " ".join(list(match.groups()))
else:
return text
return(items )
model = Sequential()
l2_reg_conv2d = 0
l2_reg_dense = 0.01
activation_type = 'relu'
model.add(Conv2D(64, kernel_size=3, activation=activation_type, input_shape=(28, 28, 1), padding='same', kernel_regularizer=regularizers.l2(l2_reg_conv2d)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=3, activation=activation_type, padding='same', kernel_regularizer=regularizers.l2(l2_reg_conv2d)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=5, activation=activation_type, padding='same', kernel_regularizer=regularizers.l2(l2_reg_conv2d)))
 model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(3, 3)))
 model.add(Dropout(0.4))
model.add(Conv2D(128, kernel_size=3, activation=activation_type, padding='same', kernel_regularizer=regularizers.l2(l2_reg_conv2d)))
 model.add(BatchNormalization())
 model.add(Conv2D(128, kernel_size=3, activation=activation_type, padding='same', kernel_regularizer=regularizers.l2(l2_reg_conv2d)))
 model.add(BatchNormalization())
 model.add(Conv2D(128, kernel_size=5, activation=activation_type, padding='same', kernel_regularizer=regularizers.l2(l2_reg_conv2d)))
 model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(3, 3)))
 model.add(Dropout(0.3))
model.add(Conv2D(256, kernel_size=3, activation=activation_type, padding='same', kernel_regularizer=regularizers.l2(l2_reg_conv2d)))
 model.add(BatchNormalization())
 model.add(Conv2D(256, kernel_size=3, activation=activation_type, padding='same', kernel_regularizer=regularizers.l2(l2_reg_conv2d)))
 model.add(BatchNormalization())
 model.add(Conv2D(256, kernel_size=5, activation=activation_type, padding='same', kernel_regularizer=regularizers.l2(l2_reg_conv2d)))
 model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(3, 3)))
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256, activation=activation_type, kernel_regularizer=regularizers.l2(l2_reg_dense)))
 model.add(BatchNormalization())
 model.add(Dense(128, activation=activation_type, kernel_regularizer=regularizers.l2(l2_reg_dense)))
 model.add(BatchNormalization())
 model.add(Dense(10, activation='softmax'))
model.compile(optimizer = SGD(lr=0.01),
loss = 'sparse_categorical_crossentropy',
metrics=['accuracy'])
 model.summary()<init_hyperparams>
def clean_text(text):
text = re.sub(r"%20", " ", text)
 text = text.replace(r"@", " ")
 text = text.replace(r"
text = text.replace(r"'", " ")
 text = text.replace(r"\x89û_", " ")
 text = text.replace(r"??????", " ")
 text = text.replace(r"\x89ûò", " ")
 text = text.replace(r"16yr", "16 year")
 text = text.replace(r"re\x89û_", " ")
 text = text.replace(r"mh370", " ")
 text = text.replace(r"prebreak", "pre break")
 text = re.sub(r"\x89û", " ", text)
 text = re.sub(r"re\x89û", "re ", text)
 text = text.replace(r"nowplaying", "now playing")
 text = re.sub(r"\x89ûª", "'", text)
 text = re.sub(r"\x89û", " ", text)
 text = re.sub(r"\x89ûò", " ", text)
 text = re.sub(r"\x89Û_", "", text)
 text = re.sub(r"\x89ÛÒ", "", text)
 text = re.sub(r"\x89ÛÓ", "", text)
 text = re.sub(r"\x89ÛÏWhen", "When", text)
 text = re.sub(r"\x89ÛÏ", "", text)
 text = re.sub(r"China\x89Ûªs", "China's", text)
 text = re.sub(r"let\x89Ûªs", "let's", text)
 text = re.sub(r"\x89Û÷", "", text)
 text = re.sub(r"\x89Ûª", "", text)
 text = re.sub(r"\x89Û\x9d", "", text)
 text = re.sub(r"å_", "", text)
 text = re.sub(r"\x89Û¢", "", text)
 text = re.sub(r"\x89Û¢åÊ", "", text)
 text = re.sub(r"fromåÊwounds", "from wounds", text)
 text = re.sub(r"åÊ", "", text)
 text = re.sub(r"åÈ", "", text)
 text = re.sub(r"JapÌ_n", "Japan", text)
 text = re.sub(r"Ì©", "e", text)
 text = re.sub(r"å¨", "", text)
 text = re.sub(r"SuruÌ¤", "Suruc", text)
 text = re.sub(r"åÇ", "", text)
 text = re.sub(r"å£3million", "3 million", text)
 text = re.sub(r"åÀ", "", text)
 text = re.sub(r'http\S+', ' ', text)
 text = re.sub(r"ªs", " ", text)
 text = re.sub(r"ª", " ", text)
 text = re.sub(r"\x9d", " ", text)
 text = re.sub(r"ò", " ", text)
 text = re.sub(r"ªt", " ", text)
 text = re.sub(r"ó", " ", text)
 text = text.replace(r"11yearold", "11 year old")
 text = re.sub(r"typhoondevastated", "typhoon devastated", text)
 text = re.sub(r"bestnaijamade", "best nijamade", text)
 text = re.sub(r"gbbo", "The Great British Bake Off", text)
 text = re.sub(r"ï", "", text)
 text = re.sub(r"ïwhen", "when", text)
 text = re.sub(r"selfimage", "self image", text)
 text = re.sub(r"20150805", "2015 08 05", text)
 text = re.sub(r"20150806", "2015 08 06", text)
 text = re.sub(r"subreddits", "website for weird public sentiment", text)
 text = re.sub(r"disea", "chinese famous electronic company", text)
 text = re.sub(r"lmao", "funny", text)
 text = text.replace(r"companyse", "company")
 text = text.replace(r"worldnews", "world news")
 text = text.replace(r"animalrescue", "animal rescue")
 text = text.replace(r"freakiest", "freak")
 text = text.replace(r"irandeal", "iran deal")
 text = text.replace(r"directioners", "mentor")
 text = text.replace(r"justinbieber", "justin bieber")
 text = text.replace(r"okwx", "okay")
 text = text.replace(r"trapmusic", "trap music")
 text = text.replace(r"djicemoon", "music ice moon")
 text = text.replace(r"icemoon", "ice moon")
 text = text.replace(r"mtvhottest", "tv hottest")
 text = text.replace(r"rì©union", "reunion")
 text = text.replace(r"abcnews", "abc news")
 text = text.replace(r"tubestrike", "tube strike")
 text = text.replace(r"prophetmuhammad", "prophet muhammad muslim dharma")
 text = text.replace(r"chicagoarea", "chicago area")
 text = text.replace(r"yearold", "year old")
 text = text.replace(r"meatloving", "meat love")
 text = text.replace(r"standuser", "standard user")
 text = text.replace(r"pantherattack", "panther attack")
 text = text.replace(r"youngheroesid", "young hearos id")
 text = text.replace(r"idk", "i do not know")
 text = text.replace(r"usagov", "united state of america government")
 text = text.replace(r"injuryi", "injury")
 text = text.replace(r"summerfate", "summer fate")
 text = text.replace(r"kerricktrial", "kerrick trial")
 text = text.replace(r"viralspell", "viral spell")
 text = text.replace(r"collisionno", "collision")
 text = text.replace(r"socialnews", "social news")
 text = text.replace(r"nasahurricane", "nasa hurricane")
 text = text.replace(r"strategicpatience", "strategic patience")
 text = text.replace(r"explosionproof", "explosion proof")
 text = text.replace(r"selfies", "photo")
 text = text.replace(r"selfie", "photo")
 text = text.replace(r"worstsummerjob", "worst summer job")
 text = text.replace(r"realdonaldtrump", "real america president")
 text = text.replace(r"omfg", "oh my god")
 text = text.replace(r"japìn", "japan")
 text = text.replace(r"breakingnews", "breaking news")
 text = " ".join([split_textnum(word)for word in text.split(" ")])
 text = "".join([c if c not in string.punctuation else "" for c in text])
 text = ''.join(c for c in text if not c.isdigit())
 text = text.replace(r"÷", "")
 text = re.sub(' +', ' ', text)
 return text
datagen = ImageDataGenerator(
 rotation_range = 20,
width_shift_range = 0.3,
height_shift_range = 0.3,
shear_range = 0.2,
zoom_range = 0.3,
horizontal_flip = False )<train_model>
%%time
train.text = train.text.apply(lambda x : clean_text(x))
test.text = test.text.apply(lambda x : clean_text(x))
train.text = train.text.apply(lambda x : " ".join([contraction_mapping[word].lower() if word in contraction_mapping.keys() else word.lower() for word in x.split(" ")]))
test.text = test.text.apply(lambda x : " ".join([contraction_mapping[word].lower() if word in contraction_mapping.keys() else word.lower() for word in x.split(" ")]))
epochs = 75
batch_size = 128
X_train = X_train.reshape(X_train.shape[0],28,28,1)
 X_test = X_test.reshape(X_test.shape[0],28,28,1)
 train_story = model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size),
epochs = epochs,
steps_per_epoch = 100,
validation_data =(X_test, y_test),
callbacks=[
ModelCheckpoint('/kaggle/working/best_kannada_model.h5', save_best_only=True),
CSVLogger('/kaggle/working/learning_log_RMSprop_without_BN.csv'),
],
verbose=1 )<load_from_csv>
vocab_train = build_vocab(train['text'])
 print("Glove : Train")
 oov_glove_train = check_coverage(vocab_train, embed_glove)
 vocab_test = build_vocab(test['text'])
 print("Glove : Test")
 oov_glove_test = check_coverage(vocab_test, embed_glove )
log_batch_norm = np.array(pd.read_csv("/kaggle/working/learning_log_RMSprop_with_BN.csv")['val_accuracy'])
 log_no_batch_norm = np.array(pd.read_csv("/kaggle/working/learning_log_RMSprop_without_BN.csv")['val_accuracy'])
 plt.figure(figsize=(20,10))
plt.plot(range(1, 11), log_batch_norm, label='with BatchNorm')
 plt.plot(range(1, 11), log_no_batch_norm, label='without BatchNorm')
 plt.legend()
plt.show()<load_from_csv>
lemmatizer = WordNetLemmatizer()
train.text = train.text.apply(lambda x : "".join([lemmatizer.lemmatize(word)for word in x]))
test.text = test.text.apply(lambda x : "".join([lemmatizer.lemmatize(word)for word in x]))
log_softmax = np.array(pd.read_csv("/kaggle/working/learning_log_RMSprop_softmax.csv")['val_accuracy'])
 log_elu = np.array(pd.read_csv("/kaggle/working/learning_log_RMSprop_elu.csv")['val_accuracy'])
 log_relu = np.array(pd.read_csv("/kaggle/working/learning_log_RMSprop_relu.csv")['val_accuracy'])
 log_tanh = np.array(pd.read_csv("/kaggle/working/learning_log_RMSprop_tanh.csv")['val_accuracy'])
 log_sigmoid = np.array(pd.read_csv("/kaggle/working/learning_log_RMSprop_sigmoid.csv")['val_accuracy'])
 log_exponential = np.array(pd.read_csv("/kaggle/working/learning_log_RMSprop_exponential.csv")['val_accuracy'])
 plt.figure(figsize=(20,10))
plt.plot(range(1, 11), log_softmax, label='softmax')
 plt.plot(range(1, 11), log_elu, label='elu')
 plt.plot(range(1, 11), log_relu, label='relu')
 plt.plot(range(1, 11), log_tanh, label='tanh')
 plt.plot(range(1, 11), log_sigmoid, label='sigmoid')
 plt.plot(range(1, 11), log_exponential, label='exponential')
 plt.legend()
plt.show()<train_model>
vocab_train = build_vocab(train['text'])
 print("Glove : Train")
 oov_glove_train = check_coverage(vocab_train, embed_glove)
 vocab_test = build_vocab(test['text'])
 print("Glove : Test")
 oov_glove_test = check_coverage(vocab_test, embed_glove )
<load_from_csv>
del oov_glove_test
del embed_glove
gc.collect()
log_SGD = np.array(pd.read_csv("/kaggle/working/learning_log_SGD.csv")['val_accuracy'])
 log_SGD_mom = np.array(pd.read_csv("/kaggle/working/learning_log_SGD_mom.csv")['val_accuracy'])
 log_Adam = np.array(pd.read_csv("/kaggle/working/learning_log_Adam.csv")['val_accuracy'])
 log_Adadelta = np.array(pd.read_csv("/kaggle/working/learning_log_Adadelta.csv")['val_accuracy'])
 log_RMSprop = np.array(pd.read_csv("/kaggle/working/learning_log_RMSprop.csv")['val_accuracy'] )<load_from_csv>
class ClassificationReport(Callback):
def __init__(self, train_data=() , validation_data=()):
super(Callback, self ).__init__()
self.X_train, self.Y_train = train_data
self.X_val, self.Y_val = validation_data
self.train_precision_score = []
self.train_recall_score = []
self.train_f1_score = []
self.val_precision_score = []
self.val_recall_score = []
self.val_f1_score = []
def on_epoch_end(self, epoch, logs={}):
train_prediction = np.round(self.model.predict(self.X_train, verbose=0))
train_precision = precision_score(self.Y_train, train_prediction, average='macro')
 train_recall = recall_score(self.Y_train, train_prediction, average='macro')
 train_f1 = f1_score(self.Y_train, train_prediction, average='macro')
 self.train_precision_score.append(train_precision)
 self.train_recall_score.append(train_recall)
 self.train_f1_score.append(train_f1)
 val_prediction = np.round(self.model.predict(self.X_val, verbose=0))
val_precision = precision_score(self.Y_val, val_prediction, average='macro')
 val_recall = recall_score(self.Y_val, val_prediction, average='macro')
 val_f1 = f1_score(self.Y_val, val_prediction, average='macro')
 self.val_precision_score.append(val_precision)
 self.val_recall_score.append(val_recall)
 self.val_f1_score.append(val_f1)
 print('
Epoch - {} - Training Precision - {:.6} - Training Recall - {:.6} - Training F1-Score - {:.6}'.format(
 epoch+1, train_precision, train_recall, train_f1))
print('
Epoch - {} - Validation Precision - {:.6} - Validation Recall - {:.6} - Validation F1-Score - {:.6}'.format(
 epoch+1, val_precision, val_recall, val_f1))
test_csv = pd.read_csv(".. /input/Kannada-MNIST/test.csv")
 X_val = np.array(test_csv.drop("id",axis=1), dtype=np.float32)
 X_val.shape<predict_on_test>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
bert_layer = hub.KerasLayer('https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1', trainable=True )
best_model = load_model('/kaggle/working/best_kannada_model.h5')
 Y_val = best_model.predict(X_val)
 Y_val = np.argmax(Y_val, axis = 1 )<save_to_csv>
class BertTraining:
def __init__(self, bert_layer, fold_k=2, dropout=0.2, max_seq_len=160, lr=0.0001, epochs=15, batch_size=32):
self.fold_k = fold_k
self.bert_layer = bert_layer
self.max_seq_len = max_seq_len
self.lr = lr
self.dropout = dropout
self.epochs = epochs
self.batch_size = batch_size
self.models = []
self.scores = {}
vocab_file = self.bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = self.bert_layer.resolved_object.do_lower_case.numpy()
self.tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 def bert_encode(self, texts):
all_token_ids = []
all_masks = []
all_segments = []
for text in texts:
text = self.tokenizer.tokenize(text)
 text = text[: self.max_seq_len-2]
input_seqence = ['[CLS]'] + text + ['[SEP]']
padding_length = self.max_seq_len - len(input_seqence)
 tokens = self.tokenizer.convert_tokens_to_ids(input_seqence)
 tokens = tokens + [0]*padding_length
pad_masks = [1]*len(input_seqence)+ [0]*padding_length
segment_ids = [0]*self.max_seq_len
all_token_ids.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_token_ids), np.array(all_masks), np.array(all_segments)
 def bert_model(self):
input_token_id = Input(shape=(self.max_seq_len,), dtype=tf.int32, name='input_token_id')
 input_mask = Input(shape=(self.max_seq_len,), dtype=tf.int32, name='input_mask')
 input_segment = Input(shape=(self.max_seq_len,), dtype=tf.int32, name='input_segment')
 _, sequence_output = self.bert_layer([input_token_id, input_mask, input_segment])
 clf_output = sequence_output[:, 0, :]
if self.dropout == 0:
output = Dense(1, activation='sigmoid' )(clf_output)
 else:
dropout = Dropout(self.dropout )(clf_output)
 output = Dense(1, activation='sigmoid' )(dropout)
 model = Model(inputs=[input_token_id, input_mask, input_segment], outputs=output)
 optimizer = SGD(learning_rate=self.lr, momentum=0.8)
 model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])
 return model
def train_model(self, X):
skf = StratifiedKFold(n_splits=self.fold_k, random_state=SEED, shuffle=True)
 for fold,(train_idx, val_idx)in enumerate(skf.split(X.text, X.text)) :
print("
Fold {}
".format(fold))
X_Train_Encoded = self.bert_encode(X.loc[train_idx, 'text'].str.lower())
 Y_Train = X.loc[train_idx, 'target']
X_Val_Encoded = self.bert_encode(X.loc[val_idx, 'text'].str.lower())
 Y_Val = X.loc[val_idx, 'target']
metrics = ClassificationReport(train_data=(X_Train_Encoded, Y_Train), validation_data=(X_Val_Encoded, Y_Val))
model = self.bert_model()
model.fit(X_Train_Encoded, Y_Train, epochs=self.epochs, batch_size=self.batch_size, callbacks=[metrics],
validation_data=(X_Val_Encoded, Y_Val), verbose=1)
 self.models.append(model)
 self.scores[fold] = {
'train' : {
'precision' : metrics.train_precision_score,
'recall' : metrics.train_recall_score,
'f1_score': metrics.train_f1_score
},
'validation' : {
'precision' : metrics.val_precision_score,
'recall' : metrics.val_recall_score,
'f1_score': metrics.val_f1_score
}
}
def plot_learning_curve(self):
fig, axes = plt.subplots(nrows=self.fold_k, ncols=2, figsize=(20, self.fold_k * 6), dpi=100)
 for i in range(self.fold_k):
sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.models[i].history.history['val_accuracy'], ax=axes[i][0], label='val_accuracy')
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.scores[i]['validation']['precision'], ax=axes[i][0], label='val_precision')
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.scores[i]['validation']['recall'], ax=axes[i][0], label='val_recall')
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.scores[i]['validation']['f1_score'], ax=axes[i][0], label='val_f1')
 axes[i][0].legend()
axes[i][0].set_title('Fold {} Validation Classification Report'.format(i), fontsize=14)
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.models[0].history.history['loss'], ax=axes[i][1], label='train_loss')
 sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.models[0].history.history['val_loss'], ax=axes[i][1], label='val_loss')
 axes[i][1].legend()
axes[i][1].set_title('Fold {} Train / Validation Loss'.format(i), fontsize=14)
 for j in range(2):
axes[i][j].set_xlabel('Epoch', size=12)
 axes[i][j].tick_params(axis='x', labelsize=12)
 axes[i][j].tick_params(axis='y', labelsize=12)
 plt.show()
def predict(self, X_test):
X_test_encode = self.bert_encode(X_test.text.str.lower())
 Y_pred = np.zeros(( X_test_encode[0].shape[0], 1))
for model in self.models:
Y_pred += model.predict(X_test_encode)/len(self.models)
 return Y_pred
submission = pd.read_csv(".. /input/Kannada-MNIST/sample_submission.csv")
 submission['label'] = Y_val
submission.to_csv("submission.csv",index=False )<import_modules>
SEED = 42
clf = BertTraining(bert_layer, fold_k=3, dropout=0.5, max_seq_len=140, lr=0.0001, epochs=20, batch_size=64)
 clf.train_model(train )
import pandas as pd
import numpy as np
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn import metrics
import time
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.layers import Dense, Conv2D, Flatten, MaxPool2D, Dropout, BatchNormalization, LeakyReLU
from tensorflow.keras.layers import BatchNormalization
from tensorflow.keras.callbacks import LearningRateScheduler
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler
import matplotlib.pyplot as plt<load_from_csv>
prediction = clf.predict(test)
 prediction
train_data = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')
 test_data = pd.read_csv("/kaggle/input/Kannada-MNIST/test.csv")
 dig_data = pd.read_csv("/kaggle/input/Kannada-MNIST/Dig-MNIST.csv" )<prepare_x_and_y>
prediction = np.where(prediction < 0.5, 0, 1)
 prediction
data_train = train_data.iloc[:,1:].values
x_train = data_train.reshape(data_train.shape[0], 28, 28, 1)
 train_label = train_data.iloc[:,0].values
y_train = tf.keras.utils.to_categorical(train_label, 10)
 print(x_train.shape, y_train.shape )<categorify>
result = pd.DataFrame()
result["id"] = test['id']
result["target"] = np.squeeze(prediction)
 result.head()
data_val=dig_data.drop('label',axis=1 ).iloc[:,:].values
x_val = data_val.reshape(data_val.shape[0], 28, 28,1)
 val_label=dig_data.label
y_val = tf.keras.utils.to_categorical(val_label, 10)
 print(x_val.shape, y_val.shape )<choose_model_class>
result.target.value_counts()
<choose_model_class><EOS> 
result.to_csv('submission.csv', index=False )