Kaggle competition: Quora Insincere Questions Classification
import numpy as np, pandas as pd
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler, PolynomialFeatures
from sklearn.feature_selection import VarianceThreshold
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis<load_from_csv>
import pandas as pd
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import keras
import operator
print('Loading Train')
 train = pd.read_csv('.. /input/train.csv')
 print('Loading Test')
 test = pd.read_csv('.. /input/test.csv')
 print('Finish' )<normalization>
df_train=pd.read_csv('.. /input/train.csv')
 df_test=pd.read_csv('.. /input/test.csv' )
oof = np.zeros(len(train))
preds = np.zeros(len(test))
oof_QDA = np.zeros(len(train))
preds_QDA = np.zeros(len(test))
cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
for i in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 poly = PolynomialFeatures(degree=2)
 sc = StandardScaler()
data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = sc.fit_transform(poly.fit_transform(VarianceThreshold(threshold=1.5 ).fit_transform(data[cols])))
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = VarianceThreshold(threshold=1.5 ).fit_transform(data[cols])
 train4 = data2[:train2.shape[0]]; test4 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = LogisticRegression(solver='saga',penalty='l2',C=0.01,tol=0.001)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
clf_QDA = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf_QDA.fit(train4[train_index,:],train2.loc[train_index]['target'])
 oof_QDA[idx1[test_index]] = clf_QDA.predict_proba(train4[test_index,:])[:,1]
preds_QDA[idx2] += clf_QDA.predict_proba(test4)[:,1] / skf.n_splits
if i%64==0:
print(i, 'LR oof auc : ', round(roc_auc_score(train['target'][idx1], oof[idx1]), 5))
print(i, 'QDA oof auc : ', round(roc_auc_score(train['target'][idx1], oof_QDA[idx1]), 5))<feature_engineering>
print("Train data target 1")
 print(df_train[df_train['target']==1].count())
 print("Train data target 0")
 print(df_train[df_train['target']==0].count() )
test['target'] = preds
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for k in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2p = test2[(test2['target']<=0.01)|(test2['target']>=0.99)].copy()
test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train3p = sel.transform(train2p[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 poly = PolynomialFeatures(degree=2 ).fit(train3p)
 train3p = poly.transform(train3p)
 train3 = poly.transform(train3)
 test3 = poly.transform(test3)
 sc2 = StandardScaler()
train3p = sc2.fit_transform(train3p)
 train3 = sc2.transform(train3)
 test3 = sc2.transform(test3)
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[ test_index<len(train3)]
clf = LogisticRegression(solver='saga',penalty='l2',C=0.01,tol=0.001)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof[idx1[test_index3]] = clf.predict_proba(train3[test_index3,:])[:,1]
preds[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
if k%64==0:
print(k, 'LR2 oof auc : ', round(roc_auc_score(train['target'][idx1], oof[idx1]), 5))<feature_engineering>
all_phrases=df_train[df_train.target != 2]
all_words = []
for t in all_phrases.question_text:
all_words.append(t)
 all_words[:4]
test['target'] = preds_QDA
oof_QDA2 = np.zeros(len(train))
preds_QDA2 = np.zeros(len(test))
for k in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2p = test2[(test2['target']<=0.01)|(test2['target']>=0.99)].copy()
test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train3p = sel.transform(train2p[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[ test_index<len(train3)]
clf_QDA2 = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf_QDA2.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_QDA2[idx1[test_index3]] = clf_QDA2.predict_proba(train3[test_index3,:])[:,1]
preds_QDA2[test2.index] += clf_QDA2.predict_proba(test3)[:,1] / skf.n_splits
if k%64==0:
print(k, 'QDA2 oof auc : ', round(roc_auc_score(train['target'][idx1], oof_QDA2[idx1]), 5))<compute_test_metric>
all_text = pd.Series(all_words ).str.cat(sep=' ' )
print('LR auc: ', round(roc_auc_score(train['target'], oof),5))
print('QDA auc: ', round(roc_auc_score(train['target'], oof_QDA2),5))<compute_test_metric>
wordcloud = WordCloud(width=1600, height=800, max_font_size=200 ).generate(all_text)
 plt.figure(figsize=(12,10))
plt.imshow(wordcloud, interpolation='bilinear')
 plt.axis("off")
 plt.show()
w_best = 0
oof_best = oof_QDA2
for w in np.arange(0,0.55,0.001):
oof_blend = w*oof+(1-w)*oof_QDA2
if(roc_auc_score(train['target'], oof_blend)) >(roc_auc_score(train['target'], oof_best)) :
w_best = w
oof_best = oof_blend
print(w_best)
 print('best weight: ', w_best)
 print('auc_best: ', round(roc_auc_score(train['target'], oof_best), 5))<save_to_csv>
neg_phrases = df_train[df_train.target == 1]
neg_words = []
for t in neg_phrases.question_text:
neg_words.append(t)
 neg_words[:4]
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = w_best*preds +(1-w_best)*preds_QDA2
sub.to_csv('submission.csv', index=False)
 sub.head()<set_options>
neg_text = pd.Series(neg_words ).str.cat(sep=' ')
 neg_text[:100]
warnings.filterwarnings('ignore')
<load_from_csv>
wordcloud = WordCloud(width=1600, height=800, max_font_size=200 ).generate(neg_text)
 plt.figure(figsize=(12,10))
plt.imshow(wordcloud, interpolation='bilinear')
 plt.axis("off")
 plt.show()
%%time
path=Path('.. /input')
 def load_data(data):
return pd.read_csv(data)
 with multiprocessing.Pool() as pool:
train, test, sub = pool.map(load_data, [path/'train.csv',
path/'test.csv',
path/'sample_submission.csv'] )<init_hyperparams>
pos_phrases = df_train[df_train.target == 0]
pos_words = []
for t in pos_phrases.question_text:
pos_words.append(t)
 pos_words[:4]
NFOLDS=5
NTRIALS=100
RS=42
debug=0
lowest=0.01
highest=0.99<define_variables>
pos_text = pd.Series(pos_words ).str.cat(sep=' ')
 pos_text[:100]
if debug:
magic_max=2
magic_min=0
NFOLDS=2
NTRIALS=2
else:
magic_max=train['wheezy-copper-turtle-magic'].max()
magic_min=train['wheezy-copper-turtle-magic'].min()<prepare_x_and_y>
wordcloud = WordCloud(width=1600, height=800, max_font_size=200 ).generate(pos_text)
 plt.figure(figsize=(12,10))
plt.imshow(wordcloud, interpolation='bilinear')
 plt.axis("off")
 plt.show()
def preprocess(clfs=['QDA'], train=train, test=test, magic_min=magic_min, magic_max=magic_max):
prepr = {}
for i in range(magic_min, magic_max+1):
X = train[train['wheezy-copper-turtle-magic']==i].copy()
Y = X.pop('target' ).values
X_test = test[test['wheezy-copper-turtle-magic']==i].copy()
idx_train = X.index
idx_test = X_test.index
X.reset_index(drop=True,inplace=True)
 cols = [c for c in X.columns if c not in ['id', 'wheezy-copper-turtle-magic']]
l=len(X)
 X_all = pd.concat([X[cols], X_test[cols]], ignore_index=True)
 X_vt = VarianceThreshold(threshold=1.5 ).fit_transform(X_all)
 prepr['vt_' + str(i)] = X_vt
prepr['train_size_' + str(i)] = l
prepr['idx_train_' + str(i)] = idx_train
prepr['idx_test_' + str(i)] = idx_test
prepr['target_' + str(i)] = Y
return prepr<categorify>
df_train['length'] = df_train['question_text'].str.count(' ')+ 1
df_test['length'] = df_test['question_text'].str.count(' ')+ 1
%%time
data = preprocess()<prepare_x_and_y>
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def check_coverage(vocab, embeddings_index):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.3%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.3%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
def get_data(i, data):
l = data['train_size_' + str(i)]
X_all = data['vt_' + str(i)]
X = X_all[:l, :]
X_test = X_all[l:, :]
Y = data['target_' + str(i)]
idx_train = data['idx_train_' + str(i)]
idx_test = data['idx_test_' + str(i)]
return X, X_test, Y, idx_train, idx_test<train_model>
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding" )
def pseudolabeling(X_train, X_test, Y_train, Y_pseudo,
idx_test, lowest=lowest, highest=highest, test=test):
assert len(test)== len(Y_pseudo), "The length of test does not match that of Y_pseudo!"
Y_aug = Y_pseudo[idx_test]
assert len(Y_aug)== len(X_test), "The length of Y_aug does not match that of X_test!"
Y_aug[Y_aug > highest] = 1
Y_aug[Y_aug < lowest] = 0
mask =(Y_aug == 1)|(Y_aug == 0)
 Y_useful = Y_aug[mask]
X_test_useful = X_test[mask]
X_train_aug = np.vstack(( X_train, X_test_useful))
Y_train_aug = np.vstack(( Y_train.reshape(-1, 1), Y_useful.reshape(-1, 1)))
 return X_train_aug, Y_train_aug<train_model>
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
punct = "/-'?!.,
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
mispell_dict = {'examinaton': 'examination',
'undergraduation': 'under graduation',
'fiancé': 'fiance',
'qoura': 'quora',
'bhakts': 'followers',
'quorans': 'quora users',
'brexit': 'Britain exit',
'cryptocurrencies': 'cryptocurrency',
'colour': 'color',
'centre': 'center',
'favourite': 'favorite',
'travelling': 'traveling',
'counselling': 'counseling',
'theatre': 'theater',
'cancelled': 'canceled',
'labour': 'labor',
'organisation': 'organization',
'wwii': 'world war 2',
'citicise': 'criticize',
'youtu ': 'youtube ',
'Qoura': 'Quora',
'sallary': 'salary',
'Whta': 'What',
'narcisist': 'narcissist',
'howdo': 'how do',
'whatare': 'what are',
'howcan': 'how can',
'howmuch': 'how much',
'howmany': 'how many',
'whydo': 'why do',
'doI': 'do I',
'theBest': 'the best',
'howdoes': 'how does',
'mastrubation': 'masturbation',
'mastrubate': 'masturbate',
"mastrubating": 'masturbating',
'pennis': 'penis',
'Etherium': 'Ethereum',
'narcissit': 'narcissist',
'bigdata': 'big data',
'2k17': '2017',
'2k18': '2018',
'qouta': 'quota',
'exboyfriend': 'ex boyfriend',
'airhostess': 'air hostess',
"whst": 'what',
'watsapp': 'whatsapp',
'demonitisation': 'demonetization',
'demonitization': 'demonetization',
'demonetisation': 'demonetization',
'pokémon': 'pokemon',
'paralizing': 'paralising',
'perfeccionism': 'perfectionism',
'depreciaton': 'depreciation',
'abvicable': 'abdicable',
'catanation': 'catenation',
'leasership': 'leadership',
'webassembly': 'web assembly',
'fortitide': 'fortitude',
'withdrow': 'withdraw',
'bomblasts': 'bomb blasts',
'engineerer': 'engineer',
'citycarclean': 'city car clean',
'billionsites': 'billion sites',
'willhandjob': 'will hand job',
'fireguns': 'fire guns',
'justeat': 'just eat',
'ubereats': 'uber eats',
'doinformation': 'do information',
'freshersworld': 'freshers world',
'topicwise': 'topic wise',
'excitee': 'excited',
'bengalore': 'bangalore',
'proproetor': 'proprietor',
'migeration': 'migration',
'ejectulate': 'Ejaculate',
'glucoze': 'glucose',
'whatapp': 'whatsapp',
'sumup': 'sum up',
'besic': 'basic',
'experienceed': 'experienced',
'feminisam': 'feminism',
'kayboard': 'keyboard',
'retructuring': 'restructuring',
'becomd': 'become',
'preidct': 'predict',
'statups': 'startups',
'superbrains': 'super brains',
'becoome': 'become',
'gwroth': 'growth',
'wakeupnow': 'wake up now',
'headpone': 'headphone',
'industiry': 'industry',
'arichtecture': 'architecture',
'simlarity': 'similarity',
'walmartlabs': 'walmart labs',
'thunderstike': 'thunder stike',
'maintanable': 'maintainable',
'diffferently': 'differently',
'careamics': 'ceramics',
'sinnister': 'sinister',
'quoras': 'quora',
'breakimg': 'breaking',
'surggery': 'surgery',
'whatwill': 'what will',
'adhaar': 'identity',
'aidentity': 'identity',
'upwork': 'up work',
'alshamsi': 'al shamsi',
'litecoin': 'cryptocurrency ',
'chapterwise': 'chapter wise',
'blockchains': 'blockchain',
'flipcart': 'flipkart',
'Terroristan': 'terrorist Pakistan',
'terroristan': 'terrorist Pakistan',
'BIMARU': 'Bihar, Madhya Pradesh, Rajasthan, Uttar Pradesh',
'Hinduphobic': 'Hindu phobic',
'hinduphobic': 'Hindu phobic',
'Hinduphobia': 'Hindu phobic',
'hinduphobia': 'Hindu phobic',
'Babchenko': 'Arkady Arkadyevich Babchenko faked death',
'Boshniaks': 'Bosniaks',
'Dravidanadu': 'Dravida Nadu',
'mysoginists': 'misogynists',
'MGTOWS': 'Men Going Their Own Way',
'mongloid': 'Mongoloid',
'unsincere': 'insincere',
'meninism': 'male feminism',
'jewplicate': 'jewish replicate',
'unoin': 'Union',
'daesh': 'Islamic State of Iraq and the Levant',
'Kalergi': 'Coudenhove-Kalergi',
'Bhakts': 'Bhakt',
'bhakts': 'Bhakt',
'Tambrahms': 'Tamil Brahmin',
'Pahul': 'Amrit Sanskar',
'SJW': 'social justice warrior',
'SJWs': 'social justice warrior',
'incel': ' involuntary celibates',
'incels': ' involuntary celibates',
'emiratis': 'Emiratis',
'weatern': 'western',
'westernise': 'westernize',
'Pizzagate': 'Pizzagate conspiracy theory',
'naïve': 'naive',
'Skripal': 'Sergei Skripal',
'Remainers': 'British remainer',
'remainers': 'British remainer',
'bremainer': 'British remainer',
'antibrahmin': 'anti Brahminism',
'HYPSM': 'Harvard, Yale, Princeton, Stanford, MIT',
'HYPS': 'Harvard, Yale, Princeton, Stanford',
'kompromat': 'compromising material',
'Tharki': 'pervert',
'tharki': 'pervert',
'mastuburate': 'masturbate',
'Zoë': 'Zoe',
'indans': 'Indian',
'xender': 'gender',
'Naxali ': 'Naxalite ',
'Naxalities': 'Naxalites',
'Bathla': 'Namit Bathla',
'Mewani': 'Indian politician Jignesh Mevani',
'clichéd': 'cliche',
'cliché': 'cliche',
'clichés': 'cliche',
'Wjy': 'Why',
'Fadnavis': 'Indian politician Devendra Fadnavis',
'Awadesh': 'Indian engineer Awdhesh Singh',
'Awdhesh': 'Indian engineer Awdhesh Singh',
'Khalistanis': 'Sikh separatist movement',
'madheshi': 'Madheshi',
'BNBR': 'Be Nice, Be Respectful',
'Bolsonaro': 'Jair Bolsonaro',
'XXXTentacion': 'Tentacion',
'Padmavat': 'Indian Movie Padmaavat',
'Žižek': 'Slovenian philosopher Slavoj Žižek',
'Adityanath': 'Indian monk Yogi Adityanath',
'Brexit': 'British Exit',
'Brexiter': 'British Exit supporter',
'Brexiters': 'British Exit supporters',
'Brexiteer': 'British Exit supporter',
'Brexiteers': 'British Exit supporters',
'Brexiting': 'British Exit',
'Brexitosis': 'British Exit disorder',
'brexit': 'British Exit',
'brexiters': 'British Exit supporters',
'jallikattu': 'Jallikattu',
'fortnite': 'Fortnite ',
'Swachh': 'Swachh Bharat mission campaign ',
'Quorans': 'Quoran',
'Qoura ': 'Quora ',
'quoras': 'Quora',
'Quroa': 'Quora',
'QUORA': 'Quora',
'narcissit': 'narcissist',
'Doklam': 'Tibet',
'Drumpf': 'Donald Trump fool',
'Drumpfs': 'Donald Trump fools',
'Strzok': 'Hillary Clinton scandal',
'rohingya': 'Rohingya ',
'wumao': 'cheap Chinese stuff',
'wumaos': 'cheap Chinese stuff',
'Sanghis': 'Sanghi',
'Tamilans': 'Tamils',
'biharis': 'Biharis',
'Rejuvalex': 'hair growth formula',
'Feku': 'Fake',
'deplorables': 'deplorable',
'muhajirs': 'Muslim immigrant',
'Gujratis': 'Gujarati',
'Chutiya': 'Fucker',
'Chutiyas': 'Fucker',
'thighing': 'masturbate',
'卐': 'Nazi Germany',
'Pribumi': 'Native Indonesian',
'Gurmehar': 'Gurmehar Kaur Indian student activist',
'Novichok': 'Soviet Union agents',
'Khazari': 'Khazars',
'Demonetization': 'demonetization',
'demonetisation': 'demonetization',
'demonitisation': 'demonetization',
'demonitization': 'demonetization',
'demonetisation': 'demonetization',
'cryptocurrencies': 'cryptocurrency',
'Hindians': 'North Indian who hate British',
'vaxxer': 'vocal nationalist ',
'remoaner': 'remainer ',
'bremoaner': 'British remainer ',
'Jewism': 'Judaism',
'Eroupian': 'European',
'WMAF': 'White male married Asian female',
'moeslim': 'Muslim',
'cishet': 'cisgender and heterosexual person',
'Eurocentric': 'Eurocentrism ',
'Jewdar': 'Jew dar',
'Asifa': 'abduction, rape, murder case ',
'marathis': 'Marathi',
'Trumpanzees': 'Trump chimpanzee fool',
'Crimean': 'Crimea people ',
'atrracted': 'attract',
'LGBT': 'lesbian, gay, bisexual, transgender',
'Boshniak': 'Bosniaks ',
'Myeshia': 'widow of Green Beret killed in Niger',
'demcoratic': 'Democratic',
'raaping': 'rape',
'Dönmeh': 'Islam',
'feminazism': 'feminism nazi',
'langague': 'language',
'Hongkongese': 'HongKong people',
'hongkongese': 'HongKong people',
'Kashmirians': 'Kashmirian',
'Chodu': 'fucker',
'penish': 'penis',
'micropenis': 'tiny penis',
'Madridiots': 'Real Madrid idiot supporters',
'Ambedkarite': 'Dalit Buddhist movement ',
'ReleaseTheMemo': 'cry for the right and Trump supporters',
'harrase': 'harass',
'Barracoon': 'Black slave',
'Castrater': 'castration',
'castrater': 'castration',
'Rapistan': 'Pakistan rapist',
'rapistan': 'Pakistan rapist',
'Turkified': 'Turkification',
'turkified': 'Turkification',
'Dumbassistan': 'dumb ass Pakistan',
'facetards': 'Facebook retards',
'rapefugees': 'rapist refugee',
'superficious': 'superficial',
'colour': 'color',
'centre': 'center',
'favourite': 'favorite',
'travelling': 'traveling',
'counselling': 'counseling',
'theatre': 'theater',
'cancelled': 'canceled',
'labour': 'labor',
'organisation': 'organization',
'wwii': 'world war 2',
'citicise': 'criticize',
'youtu ': 'youtube ',
'sallary': 'salary',
'Whta': 'What',
'narcisist': 'narcissist',
'narcissit': 'narcissist',
'howdo': 'how do',
'whatare': 'what are',
'howcan': 'how can',
'howmuch': 'how much',
'howmany': 'how many',
'whydo': 'why do',
'doI': 'do I',
'theBest': 'the best',
'howdoes': 'how does',
'mastrubation': 'masturbation',
'mastrubate': 'masturbate',
'mastrubating': 'masturbating',
'pennis': 'penis',
'Etherium': 'Ethereum',
'bigdata': 'big data',
'2k17': '2017',
'2k18': '2018',
'qouta': 'quota',
'exboyfriend': 'ex boyfriend',
'airhostess': 'air hostess',
'whst': 'what',
'watsapp': 'whatsapp',
'bodyshame': 'body shaming',
'bodyshoppers': 'body shopping',
'bodycams': 'body cams',
'Cananybody': 'Can any body',
'deadbody': 'dead body',
'deaddict': 'de addict',
'Northindian': 'North Indian ',
'northindian': 'north Indian ',
'northkorea': 'North Korea',
'Whykorean': 'Why Korean',
'koreaboo': 'Korea boo ',
'Brexshit': 'British Exit bullshit',
'shithole': 'shithole ',
'shitpost': 'shit post',
'shitslam': 'shit Islam',
'shitlords': 'shit lords',
'Fck': 'Fuck',
'fck': 'fuck',
'Clickbait': 'click bait ',
'clickbait': 'click bait ',
'mailbait': 'mail bait',
'healhtcare': 'healthcare',
'trollbots': 'troll bots',
'trollled': 'trolled',
'trollimg': 'trolling',
'cybertrolling': 'cyber trolling',
'sickular': 'India sick secular ',
'suckimg': 'sucking',
'Idiotism': 'idiotism',
'Niggerism': 'Nigger',
'Niggeriah': 'Nigger'}
def train_classifier(clf_name, clfs, data=data, train=train, test=test,
debug=debug, NFOLDS=NFOLDS, RS=RS, Y_pseudo=None,
magic_min=magic_min, magic_max=magic_max,
lowest=lowest, highest=highest, verbose=1):
auc_all = np.array([])
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in range(magic_min, magic_max+1):
X, X_test, Y, idx_train, idx_test = get_data(i=i, data=data)
 folds = StratifiedKFold(n_splits=NFOLDS, random_state=RS)
 auc_folds = np.array([])
 for train_index, val_index in folds.split(X, Y):
X_train, Y_train = X[train_index, :], Y[train_index]
X_val, Y_val = X[val_index, :], Y[val_index]
if Y_pseudo is not None:
X_train_aug, Y_train_aug = pseudolabeling(X_train, X_test,
Y_train, Y_pseudo, idx_test,
lowest=lowest, highest=highest,
test=test)
 clfs[clf_name].fit(X_train_aug, Y_train_aug)
 else:
clfs[clf_name].fit(X_train, Y_train)
 oof[idx_train[val_index]] = clfs[clf_name].predict_proba(X_val)[:,1]
preds[idx_test] += clfs[clf_name].predict_proba(X_test)[:,1]/NFOLDS
auc = roc_auc_score(Y_val, oof[idx_train[val_index]])
 auc_folds = np.append(auc_folds, auc)
 auc_all = np.append(auc_all, np.mean(auc_folds))
auc_combo = roc_auc_score(train['target'].values, oof)
 auc_av = np.mean(auc_all)
 std = np.std(auc_all)/(np.sqrt(NFOLDS)*np.sqrt(magic_max+1))
if verbose:
print(f'The result summary for the {clf_name} classifier:')
 print(f'The combined CV score is {round(auc_combo, 5)}.')
 print(f'The folds average CV score is {round(auc_av, 5)}.')
 print(f'The standard deviation is {round(std, 5)}.
 ')
 return preds, auc_combo<choose_model_class>
glove = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
paragram = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
wiki_news = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 if file == '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file)if len(o)>100)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index
%%time
results = {}
results['rp']=np.array([])
 results['auc']=np.array([])
 np.random.seed(RS)
 for j in range(NTRIALS):
rp=10**(-2*np.random.rand())
 clfs_init={'QDA': QuadraticDiscriminantAnalysis(reg_param=rp)}
clfs={'QDA': QuadraticDiscriminantAnalysis(reg_param=rp)}
Y_pseudo, _ = train_classifier('QDA', clfs=clfs_init, verbose=0)
 _, auc = train_classifier('QDA', clfs=clfs, Y_pseudo=Y_pseudo, verbose=0)
 results['rp']=np.append(results['rp'], rp)
 results['auc']=np.append(results['auc'], auc)
 print(f"Trial number {j}: AUC = {round(auc, 5)}, rp={round(rp, 5)}.
 " )<find_best_params>
glove_embeddings = load_embed(glove)
 print(len(glove_embeddings))
auc_max = np.max(results['auc'])
 i_max = np.argmax(results['auc'])
 rp_best = results['rp'][i_max]
print(f"The highest AUC achived is {round(auc_max, 5)} for rp={round(rp_best, 5)}.")
 auc_min = np.min(results['auc'])
 i_min = np.argmin(results['auc'])
 print(f"The lowest AUC achived is {round(auc_min, 5)} for rp={round(results['rp'][i_min], 5)}.")
 print(f"The smallest value of `reg_param` that was explored during the search is {round(np.min(results['rp']), 5)}.")
 print(f"The larges value of `reg_param` that was explored during the search is {round(np.max(results['rp']), 5)}." )<train_model>
train = df_train['question_text']
test = df_test['question_text']
df = pd.concat([train ,test])
 vocab = build_vocab(df )
clfs_best = {'QDA': QuadraticDiscriminantAnalysis(reg_param=rp_best)}
preds_best, auc_best = train_classifier('QDA', clfs=clfs_best, Y_pseudo=Y_pseudo, verbose=0)
 print(f"AUC: {auc_best}" )<save_to_csv>
print("oov : Glove ")
 oov = check_coverage(vocab, glove_embeddings)
 add_lower(glove_embeddings, vocab)
 print("oov : ")
 oov = check_coverage(vocab, glove_embeddings)
sub['target'] = preds_best
sub.to_csv('submission.csv',index=False )<install_modules>
def known_contractions(embed):
known = []
for contract in contraction_mapping:
if contract in embed:
known.append(contract)
 return known
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
!pip install /kaggle/input/timm-model/timm-0.3.2-py3-none-any.whl<import_modules>
def clean_special_chars(text, punct, puncts, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 for p in puncts:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
import pandas as pd
import cv2
import numpy as np
from tqdm.auto import tqdm
from matplotlib import pyplot as plt
import time
import pyarrow.parquet as pq<set_options>
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
class BengaliDataset(Dataset):
def __init__(self, label_csv, unique_csv, train_folder, parquet_path, transforms, cache=True, test=True):
self.label_csv = label_csv
self.unique_csv = unique_csv
self.train_folder = train_folder
self.parquet_path = parquet_path
self.label = pd.read_csv(self.label_csv)
 self.label = self.label[self.label['component']=='grapheme_root']
self.label = self.label.reset_index(drop=True)
 unique_df = pd.read_csv(self.unique_csv)
 self.names = self.label['image_id'].values
self.uniques = unique_df.grapheme.unique()
self.transforms = transforms
self.img = [None] * self.label.shape[0]
self.test = test
if cache:
self.cache_images()
def cache_images(self):
count = 0
root = './test_graphemes/'
self.p = p = pd.read_parquet(self.parquet_path ).drop(['image_id'], axis=1)
 def load_image(self, idx):
img = resize(self.p.iloc[[idx]] ).values.reshape(-1 ).reshape(IMG_SIZE, IMG_SIZE ).astype(np.uint8)
 return self.transforms(img)
 img = self.img[idx]
if img is None:
img = resize(p.iloc[[i]] ).values.reshape(-1 ).reshape(IMG_SIZE, IMG_SIZE ).astype(np.uint8)
 return self.transforms(img)
 else:
return self.transforms(img)
 def __getitem__(self, idx):
if not self.test:
img = self.load_image(idx)
 root = self.label.loc[idx]['grapheme_root']
consonant = self.label.loc[idx]['consonant_diacritic']
vowel = self.label.loc[idx]['vowel_diacritic']
unique = np.where(self.uniques == self.label.grapheme[idx])[0][0]
return transforms.ToTensor()(img), root, consonant, vowel, unique
else:
img = self.load_image(idx)
 root = 0
consonant = 0
vowel = 0
unique = 0
return transforms.ToTensor()(img), root, consonant, vowel, unique
def __len__(self):
return self.p.shape[0]
def postprocess(preds, num_classes, EXP = -1.2):
p0 = np.argmax(preds,axis=1)
 s = pd.Series(p0)
 vc = s.value_counts().sort_index()
df = pd.DataFrame({'a':np.arange(num_classes),'b':np.ones(num_classes)})
 df.b = df.a.map(vc)
 df.fillna(df.b.min() ,inplace=True)
 mat1 = np.diag(df.b.astype('float32')**EXP)
 p0 = np.argmax(preds.dot(mat1), axis=1)
 return p0
class Tester:
def __init__(self,
dataset_path='./drive/MyDrive/datasets/car classification/train_data',
batch_size=1,
model_name='tf_efficientnet_b3_ns',
test_csv='./train_labels.csv',
unique_csv='./train_labels.csv',
output_dir='.. /drive/MyDrive/ckpt/grapheme/submission.csv',
ckpt='.. /drive/MyDrive/ckpt/grapheme/20.pth'):
self.dataset_path = dataset_path
self.batch_size = batch_size
self.model_name = model_name
self.test_csv = test_csv
self.unique_csv = unique_csv
self.output_dir = output_dir
self.ckpt = ckpt
if model_name == 'tf_efficientnet_b0_ns':
self.input_size =(224, 224)
 elif model_name == 'tf_efficientnet_b2_ns':
self.input_size =(260, 260)
 elif model_name == 'tf_efficientnet_b3_ns':
self.input_size =(300, 300)
 elif model_name == 'tf_efficientnet_b4_ns':
self.input_size =(380, 380)
 elif model_name == 'tf_efficientnet_b6_ns':
self.input_size =(528, 528)
 else:
raise Exception('non-valid model name')
 transform = []
transform += [transforms.ToPILImage() ]
transform += [transforms.Resize(self.input_size)]
self.transform = transforms.Compose(transform)
 self.device = torch.device('cuda')
 self.model_root = create_model(self.model_name, pretrained=False, num_classes=168 ).to(self.device)
 self.model_consonant = create_model(self.model_name, pretrained=False, num_classes=8 ).to(self.device)
 self.model_vowel = create_model(self.model_name, pretrained=False, num_classes=11 ).to(self.device)
 ckpt = torch.load(self.ckpt)
 self.model_root.load_state_dict(ckpt['model_root_state_dict'])
 self.model_consonant.load_state_dict(ckpt['model_consonant_state_dict'])
 self.model_vowel.load_state_dict(ckpt['model_vowel_state_dict'])
 def test(self):
output_roots = []
output_consonants = []
output_vowels = []
count = 0
for ii in range(4):
self.test_dataset = BengaliDataset(self.test_csv, self.unique_csv, self.dataset_path, f'/kaggle/input/bengaliai-cv19/test_image_data_{ii}.parquet', self.transform, cache=True)
 self.names = self.test_dataset.names
self.test_dataloader = DataLoader(self.test_dataset, batch_size=self.batch_size, num_workers=0, shuffle=False)
 pbar = tqdm.tqdm(self.test_dataloader)
 pbar.set_description('testing process')
 self.model_root.eval()
self.model_consonant.eval()
self.model_vowel.eval()
with torch.no_grad() :
for it, data in enumerate(pbar):
inputs = data[0].to(self.device)
 inputs = inputs.repeat(1, 3, 1, 1)
 roots = data[1].to(self.device ).long()
consonants = data[2].to(self.device ).long()
vowels = data[3].to(self.device ).long()
uniques = data[4].to(self.device ).long()
root_preds = self.model_root(inputs)
 self.model_root.zero_grad()
consonant_preds = self.model_consonant(inputs)
 self.model_consonant.zero_grad()
vowel_preds = self.model_vowel(inputs)
 self.model_vowel.zero_grad()
root_preds = postprocess(torch.nn.Softmax(1 )(root_preds ).cpu().numpy() , 168, -0.6)
 consonant_preds = postprocess(torch.nn.Softmax(1 )(consonant_preds ).cpu().numpy() , 8, -0.6)
 vowel_preds = postprocess(torch.nn.Softmax(1 )(vowel_preds ).cpu().numpy() , 11, -1.1)
 for index in range(inputs.shape[0]):
output_roots.append(root_preds[index].item())
 output_consonants.append(consonant_preds[index].item())
 output_vowels.append(vowel_preds[index].item())
 del self.test_dataset.p
del self.test_dataset
del self.test_dataloader
row_id, target = [], []
for iid, r, c, v in zip(self.names, output_roots, output_consonants, output_vowels):
row_id.append(iid + '_grapheme_root')
 target.append(int(r))
row_id.append(iid + '_vowel_diacritic')
 target.append(int(v))
row_id.append(iid + '_consonant_diacritic')
 target.append(int(c))
count += 1
sub_fn = self.output_dir
sub = pd.DataFrame({'row_id': row_id, 'target': target})
 sub.to_csv(sub_fn, index=False)
 print(f'Done wrote to {sub_fn}')
 tester = Tester(dataset_path='./test_graphemes',
batch_size=128,
model_name='tf_efficientnet_b2_ns',
test_csv='/kaggle/input/bengaliai-cv19/test.csv',
unique_csv='/kaggle/input/bengaliai-cv19/train.csv',
output_dir='./submission.csv',
ckpt='/kaggle/input/graphemesingle/27.pth' 
)
 tester.test()<define_variables>
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
inputs = ["test_image_data_0.parquet","test_image_data_1.parquet","test_image_data_2.parquet","test_image_data_3.parquet"] 
<define_variables>
df_train['question_text'] = df_train['question_text'].apply(lambda x: x.lower())
 df_train['question_text'] = df_train['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))
df_train['question_text'] = df_train['question_text'].apply(lambda x: clean_special_chars(x, punct, puncts, punct_mapping))
df_train['question_text'] = df_train['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))
df_train['question_text'] = df_train['question_text'].apply(lambda x: clean_numbers(x))
DIR = ".. /input/bengaliai-cv19/"<categorify>
df_test['question_text'] = df_test['question_text'].apply(lambda x: x.lower())
 df_test['question_text'] = df_test['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))
df_test['question_text'] = df_test['question_text'].apply(lambda x: clean_special_chars(x, punct,puncts, punct_mapping))
df_test['question_text'] = df_test['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))
df_test['question_text'] = df_test['question_text'].apply(lambda x: clean_numbers(x))
class EvalDataset(Dataset):
def __init__(self,df,transform=None,aug=None,norm=None):
self.norm = norm
self.df = df
self.data = 255 - self.df.iloc[:, 1:].values.reshape(-1, 137, 236 ).astype(np.uint8)
 self.transform = transform
self.aug = aug
def __getitem__(self, idx):
name = self.df.iloc[idx,0]
image_arr = cv2.resize(self.data[idx],(128,128))
image_arr = image_arr/255.0
if self.norm != None:
mean = self.norm['mean']
std = self.norm['std']
image_arr =(image_arr - mean)/std
image_arr = torch.from_numpy(image_arr.astype(np.float32))
return image_arr.unsqueeze(0 ).repeat(3,1,1),name
def __len__(self):
return len(self.data )<set_options>
train = df_train['question_text']
test = df_test['question_text']
df = pd.concat([train ,test])
 vocab = build_vocab(df)
 print("oov : ")
 oov = check_coverage(vocab, glove_embeddings )
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
<choose_model_class>
train_df, test2_df = train_test_split(df_train, test_size=0.04, random_state=123)
 train_df, valid_df = train_test_split(train_df, test_size=0.00001, random_state=123)
 train_df=train_df.reset_index(drop=True)
 valid_df=valid_df.reset_index(drop=True)
 test2_df=test2_df.reset_index(drop=True)
 X_train=train_df['question_text'].values
y_train=train_df['target'].values
X_test2=test2_df['question_text'].values
y_test2=test2_df['target'].values
X_valid=valid_df['question_text'].values
y_valid=valid_df['target'].values
X_test=df_test['question_text'].values
print(X_train.shape)
 print(y_train.shape)
 print(X_valid.shape)
 print(y_valid.shape)
 print(X_test2.shape)
 print(y_test2.shape)
 print(X_test.shape)
 print(train_df.shape)
 print(valid_df.shape)
 print(test2_df.shape )
class MyModel(nn.Module):
def __init__(self):
super().__init__()
model = models.resnet34()
self.model = nn.Sequential(*list(model.children())[:-1])
 self.fc_g = nn.Sequential(nn.Linear(512,200), nn.ReLU() , nn.Dropout(p=0.2), nn.Linear(200,168))
self.fc_v = nn.Sequential(nn.Linear(512,200), nn.ReLU() , nn.Dropout(p=0.2), nn.Linear(200,11))
self.fc_c = nn.Sequential(nn.Linear(512,200), nn.ReLU() , nn.Dropout(p=0.2), nn.Linear(200,7))
def forward(self,x):
x = self.model(x)
 x = torch.flatten(x,1)
 g = self.fc_g(x)
 v = self.fc_v(x)
 c = self.fc_c(x)
 return g,v,c<find_best_model_class>
embedding_dim = 300
max_features = 120000
maxlen = 70
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(X_train.tolist() + X_valid.tolist() + X_test2.tolist() + X_test.tolist() )
def kaggle_out(model1,model2,model3,dataloader,device):
model1.eval()
model2.eval()
model3.eval()
grapheme_output = []
vowel_output = []
consonant_output = []
img_id_ls = []
for data in dataloader:
inputs = data[0]
image_id = data[1]
inputs = inputs.to(device)
 with torch.no_grad() :
g1,v1,c1 = model1(inputs)
 g2,v2,c2 = model2(inputs)
 g3,v3,c3 = model3(inputs)
 g = g1+g2+g3
v = v1+v2+v3
c = c1+c2+c3
grapheme_preds = g.argmax(dim=1)
 vowel_preds = v.argmax(dim=1)
 consonant_preds = c.argmax(dim=1)
 for i in range(len(image_id)) :
img_id_ls.append(image_id[i])
 grapheme_output.append(grapheme_preds[i].cpu().detach().item())
 vowel_output.append(vowel_preds[i].cpu().detach().item())
 consonant_output.append(consonant_preds[i].cpu().detach().item())
 return img_id_ls,grapheme_output,vowel_output,consonant_output<load_pretrained>
vocab_size = len(tokenizer.word_index)+ 1
print(vocab_size )
results = []
mean = 13.4/255
std = 40.8/255
model1 = MyModel()
model1.load_state_dict(torch.load('.. /input/cv-resnets/model_epoch34.pt'))
model1.to(device)
 model2 = MyModel()
model2.load_state_dict(torch.load('.. /input/cv-resnets/model_epoch69.pt'))
model2.to(device)
 model3 = MyModel()
model3.load_state_dict(torch.load('.. /input/cv-resnets/model_epoch104.pt'))
model3.to(device)
 for inp in inputs:
df = pd.read_parquet(DIR+inp)
 kaggle_dataset = EvalDataset(df,norm={'mean':mean,'std':std})
 dl = DataLoader(kaggle_dataset,batch_size=128,num_workers=4)
 img,gra,vow,con = kaggle_out(model1,model2,model3,dl,device)
 for i in range(len(img)) :
results.append(( f"{img[i]}_grapheme_root",gra[i]))
results.append(( f"{img[i]}_vowel_diacritic",vow[i]))
results.append(( f"{img[i]}_consonant_diacritic",con[i]))
df2 = pd.DataFrame(results, columns=["row_id", "target"] )<prepare_output>
def index_to_matrix(embeddings_index,word_index):
embedding_matrix = np.zeros(( len(word_index)+ 1, embedding_dim))
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return(embedding_matrix )
sub = df2<save_to_csv>
glove_embedding_matrix=index_to_matrix(glove_embeddings,tokenizer.word_index)
 embedding_matrix=glove_embedding_matrix
sub.to_csv("submission.csv", index=False )<define_variables>
from keras.models import Model
from keras.layers import Dense, Embedding, Bidirectional, CuDNNGRU,CuDNNLSTM, GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, Input, Dropout, Add
from keras.optimizers import Adam
from keras.models import Sequential
from keras import layers
import keras.callbacks
from keras.optimizers import Adam
from keras.callbacks import ModelCheckpoint
pt_models = ".. /input/pretrained-models/pretrained-models.pytorch-master/"
sys.path.insert(0, pt_models)
<categorify>
def make_model(embedding_matrix, maxlen, embed_size=300, loss='binary_crossentropy'):
inp = Input(shape=(maxlen,))
inp2 = Input(shape=(1,))
x = Embedding(vocab_size, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Dropout(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Dropout(0.2 )(x)
 avg_pl = GlobalAveragePooling1D()(x)
 max_pl = GlobalMaxPooling1D()(x)
 concat = concatenate([avg_pl, max_pl])
 dense1 = Dense(32, activation="relu" )(concat)
 concat = concatenate([dense1, inp2])
 output = Dense(1, activation="sigmoid" )(concat)
 model = Model(inputs=[inp,inp2], outputs=output)
 model.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy'])
 return model
def normalize_image(img):
return(img*(255.0/img.max())).astype(np.uint8)
 def get_min_indices(img, min_writing_value=80):
min_value = img > min_writing_value
h_min, h_max = np.where(np.any(min_value, axis=0)) [0][[0, -1]]
v_min, v_max = np.where(np.any(min_value, axis=1)) [0][[0, -1]]
return(h_min, h_max, v_min, v_max)
 def get_min_indices_with_border(img, min_writing_value=80, border=20):
h_min, h_max, v_min, v_max = get_min_indices(img[border:-border, border:-border], min_writing_value=min_writing_value)
 return(h_min + border, h_max + border, v_min + border, v_max + border)
 def cut_and_denoise_image(img, border=20, min_writing_value=80, max_noise=28):
h_min, h_max, v_min, v_max = get_min_indices_with_border(img, border=border, min_writing_value=min_writing_value)
 h_min=(h_min-border)if h_min>border else 0
v_min=(v_min-border)if v_min>border else 0
h_max=(h_max+border)if ORIGINAL_WIDTH-h_max>border else ORIGINAL_WIDTH
v_max=(v_max+border)if ORIGINAL_HEIGHT-v_max>border else ORIGINAL_HEIGHT
img = img[v_min:v_max, h_min:h_max]
img[img < max_noise] = 0
longer_side_length = max(np.ma.size(img, axis=0), np.ma.size(img, axis=1))
padding = [(( longer_side_length - np.ma.size(img, axis=0)) // 2,), 
(( longer_side_length - np.ma.size(img, axis=1)) // 2,)]
img = np.pad(img, padding, mode="constant")
 return cv2.resize(img,(PROCESSED_HEIGHT, PROCESSED_WIDTH))<load_pretrained>
model = make_model(embedding_matrix,maxlen=70 )
def load_data(file_name):
print("Dealing with ", file_name)
 df = pd.read_parquet(file_name)
 names = df.iloc[:,0].values
images = 255 - df.iloc[:,1:].values.reshape(-1, ORIGINAL_HEIGHT, ORIGINAL_WIDTH)
 del df
print("Freed after loading: ", gc.collect())
 for image_index, img in enumerate(images):
img = normalize_image(img)
 img = cut_and_denoise_image(img)
 images[image_index,0:PROCESSED_HEIGHT, 0:PROCESSED_WIDTH] = img
images = images[:,0:PROCESSED_HEIGHT, 0:PROCESSED_WIDTH]
images = images.reshape(-1, PROCESSED_HEIGHT, PROCESSED_WIDTH)
 print("Freed after processing: ", gc.collect())
 return names, images<save_to_csv>
seed = 7
n_splits=5
np.random.seed(seed)
 kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed )
with open("submission.csv", "a")as submission_file:
submission_file.write("row_id,target
")
 def write_predictions(names, prediction):
chosen_root = np.argmax(prediction[0].detach().numpy() , axis=1)
 chosen_vowel = np.argmax(prediction[1].detach().numpy() , axis=1)
 chosen_consonant = np.argmax(prediction[2].detach().numpy() , axis=1)
 with open("submission.csv", "a")as submission_file:
for index, name in enumerate(names):
submission_file.write(f"{name}_consonant_diacritic,{chosen_consonant[index]}
")
 submission_file.write(f"{name}_grapheme_root,{chosen_root[index]}
")
 submission_file.write(f"{name}_vowel_diacritic,{chosen_vowel[index]}
" )<categorify>
i=1
maxlength={}
maxl={}
for train, valid in kfold.split(X_train, y_train):
if i <=n_splits:
maxl[i]=X_len_train[train].max()
print(X_len_train[train])
 maxlength[i]=int(np.quantile(X_len_train[train],0.999))
print("Running Fold", i, "/", n_splits)
 print("split 99.9 percentile length",maxlength[i],"split max length",maxl[i])
 modelname=str("Model")+ str(i)
 train_data=[X_train[j] for j in train]
valid_data=[X_train[k] for k in valid]
train_data = tokenizer.texts_to_sequences(train_data)
 valid_data = tokenizer.texts_to_sequences(valid_data)
 train_data = pad_sequences(train_data, padding='post', maxlen=maxlength[i])
 valid_data = pad_sequences(valid_data, padding='post', maxlen=maxlength[i])
 model = make_model(embedding_matrix,maxlength[i])
 print("train", y_train[train].sum())
 print("validation",y_train[valid].sum())
 checkpointer = ModelCheckpoint(filepath=modelname,monitor='val_loss', mode='auto', verbose = 1, save_best_only=True)
 history = model.fit([train_data,X_len_train[train]], y_train[train],
epochs=5,
validation_data=([valid_data,X_len_train[valid]], y_train[valid]),
batch_size=512,callbacks=[checkpointer])
 plot_history(history)
 i=i+1
class SubmissionDataSet() :
transform = albumentations.Compose([albumentations.Normalize(always_apply=True)])
 def __init__(self, names, images):
self.names = names
self.images = images
def __len__(self):
return len(self.names)
 def __getitem__(self, index):
img = self.images[index]
img = Image.fromarray(img ).convert("RGB")
 img = self.transform(image=np.array(img)) ["image"]
img = np.transpose(img,(2,0,1))
return(self.names[index], torch.tensor(img))<choose_model_class>
y_pred={}
y_pred_test={}
count=0
for i in np.arange(1, n_splits+1, 1):
try:
model.load_weights(str("Model")+ str(i))
print(str("Model")+ str(i))
x_test2 = tokenizer.texts_to_sequences(X_test2)
 x_test = tokenizer.texts_to_sequences(X_test)
 x_test2 = pad_sequences(x_test2, padding='post', maxlen=maxlength[i])
 x_test = pad_sequences(x_test, padding='post', maxlen=maxlength[i])
 y_pred[i] = model.predict([x_test2,X_len_test2], batch_size=512, verbose=1)
 y_pred_test[i] = model.predict([x_test,X_len_test], batch_size=512, verbose=1)
 model_f1_score={}
for thresh in np.arange(0.1, 0.91, 0.01):
thresh = np.round(thresh, 2)
 model_f1_score[thresh]=sklearn.metrics.f1_score(y_test2,(y_pred[i]>=thresh ).astype(int))
model_cutoff=max(model_f1_score, key=model_f1_score.get)
 print("Max F1 score is {1} found at threshold {0}".format(model_cutoff, model_f1_score[model_cutoff]))
count=count+1
except:
pass
print('count is ', count)
class Net(torch.nn.Module):
def __init__(self):
super(Net, self ).__init__()
self.transfer_model = pretrainedmodels.__dict__["resnet101"](pretrained=None)
 self.drop1 = torch.nn.Dropout(p=0.3)
 self.l1 = torch.nn.Linear(2048, 1024)
 self.drop2 = torch.nn.Dropout(p=0.3)
 self.l2 = torch.nn.Linear(1024, 512)
 self.root = torch.nn.Linear(512, 168)
 self.vowel = torch.nn.Linear(512, 11)
 self.consonant = torch.nn.Linear(512, 7)
 def forward(self, x):
batch_size = x.shape[0]
x = self.transfer_model.features(x)
 x = torch.nn.functional.adaptive_avg_pool2d(x, 1 ).reshape(batch_size, -1)
 x = self.drop1(x)
 x = self.l1(x)
 x = self.drop2(x)
 x = self.l2(x)
 root = self.root(x)
 vowel = self.vowel(x)
 consonant = self.consonant(x)
 return(root, vowel, consonant)
 return(root, vowel, consonant)
 model = Net()
model.load_state_dict(torch.load("/kaggle/input/bengalipytorchmodels/ResNet101_30Epochs.pt", map_location=torch.device('cpu')))
 model.eval()<load_pretrained>
y_pred_final={}
y_pred_test_final={}
for i in np.arange(1, count+1, 1):
if(i == 1):
y_pred_final=y_pred[i]
y_pred_test_final=y_pred_test[i]
else:
y_pred_final=y_pred_final + y_pred[i]
y_pred_test_final=y_pred_test_final + y_pred_test[i]
y_pred_final=y_pred_final/count
y_pred_test_final=y_pred_test_final/count
for file_index in range(0, 4):
names, images = load_data(f"/kaggle/input/bengaliai-cv19/test_image_data_{file_index}.parquet")
 batch_data_set = SubmissionDataSet(names, images)
 data_loader = torch.utils.data.DataLoader(batch_data_set,
batch_size=8,
shuffle=False,
num_workers=4)
 for batch_index,(batch_names, batch_images)in enumerate(tqdm(data_loader, total=(len(batch_data_set)//data_loader.batch_size))):
prediction = model(batch_images)
 write_predictions(batch_names, prediction)
 print("Collected after writing predictions: ", gc.collect() )<install_modules>
print("Final Model on hold out")
 model_f1_score={}
for thresh in np.arange(0.1, 0.91, 0.01):
thresh = np.round(thresh, 2)
 model_f1_score[thresh]=sklearn.metrics.f1_score(y_test2,(y_pred_final>=thresh ).astype(int))
model_cutoff=max(model_f1_score, key=model_f1_score.get)
 print("Max F1 score is {1} found at threshold {0}".format(model_cutoff, model_f1_score[model_cutoff]))
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null<load_from_csv>
y_pred_test2_final_class =(y_pred_final >= model_cutoff ).astype(int)
 print(y_test2.sum())
 print(y_pred_test2_final_class.sum())
 print(sklearn.metrics.f1_score(y_test2, y_pred_test2_final_class))
print('classification report')
 print(classification_report(y_test2, y_pred_test2_final_class))
print('Confusion matrix')
 print(sklearn.metrics.confusion_matrix(y_test2, y_pred_test2_final_class))
print('roc score')
 print(sklearn.metrics.roc_auc_score(y_test2, y_pred_test2_final_class))
HEIGHT = 137
WIDTH = 236
SIZE = 128
bs = 128
stats =(0.0692, 0.2051)
 arch = pretrainedmodels.__dict__['se_resnext50_32x4d']
MODEL = '.. /input/weights/seresnext.pth'
nworkers = 2
TEST = ['/kaggle/input/bengaliai-cv19/test_image_data_0.parquet',
'/kaggle/input/bengaliai-cv19/test_image_data_1.parquet',
'/kaggle/input/bengaliai-cv19/test_image_data_2.parquet',
'/kaggle/input/bengaliai-cv19/test_image_data_3.parquet']
LABELS = '.. /input/bengaliai-cv19/train.csv'
df = pd.read_csv(LABELS)
 nunique = list(df.nunique())[1:-1]<count_unique_values>
df=pd.DataFrame(columns=['text','y_actual', 'y_pred','y_pred_prob','length'] )
nunique<choose_model_class>
df['text'] = X_test2
df['y_pred'] =y_pred_test2_final_class
df['y_pred_prob'] =y_pred_final
df['length'] =X_len_test2
df['y_actual'] =y_test2
class Head(nn.Module):
def __init__(self, nc, n, ps=0.5):
super().__init__()
layers = [AdaptiveConcatPool2d() , Mish() , Flatten() ] + \
bn_drop_lin(nc*2, 512, True, ps, Mish())+ \
bn_drop_lin(512, n, True, ps)
 self.fc = nn.Sequential(*layers)
 self._init_weight()
def _init_weight(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
torch.nn.init.kaiming_normal_(m.weight)
 elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1.0)
 m.bias.data.zero_()
def forward(self, x):
return self.fc(x)
 class Dnet_1ch(nn.Module):
def __init__(self, arch=arch, n=nunique, pre=True, ps=0.5):
super().__init__()
m = arch(pretrained='imagenet')if pre else arch(pretrained=None)
 conv = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)
 w =(m.layer0.conv1.weight.sum(1)).unsqueeze(1)
 conv.weight = nn.Parameter(w)
 self.layer0 = nn.Sequential(conv, m.layer0.bn1, m.layer0.relu1, m.layer0.pool)
 self.layer1 = m.layer1
self.layer2 = m.layer2
self.layer3 = m.layer3
self.layer4 = nn.Sequential(m.layer4[0], m.layer4[1], m.layer4[2])
 nc = self.layer4[-1].se_module.fc2.out_channels
self.head1 = Head(nc,n[0])
 self.head2 = Head(nc,n[1])
 self.head3 = Head(nc,n[2])
 def forward(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x1 = self.head1(x)
 x2 = self.head2(x)
 x3 = self.head3(x)
 return x1,x2,x3<find_best_params>
df[(df.y_actual != df.y_pred)&(df.y_pred_prob >=(model_cutoff-0.3)) &(df.y_pred_prob <=(model_cutoff + 0.3)) ]['text'].values
model = Dnet_1ch(pre=False ).cuda()
model.load_state_dict(torch.load(MODEL, map_location=torch.device('cpu')));
model.eval() ;<data_type_conversions>
wordcloud = WordCloud(width=1600, height=800, max_font_size=200 ).generate(FN_text)
 plt.figure(figsize=(12,10))
plt.imshow(wordcloud, interpolation='bilinear')
 plt.axis("off")
 plt.show()
class GraphemeDataset(Dataset):
def __init__(self, fname):
self.df = pd.read_parquet(fname)
 self.data = 255 - self.df.iloc[:, 1:].values.reshape(-1, HEIGHT, WIDTH ).astype(np.uint8)
 def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
name = self.df.iloc[idx,0]
img =(self.data[idx]*(255.0/self.data[idx].max())).astype(np.uint8)
 img = crop_resize(img)
 img =(img.astype(np.float32)/255.0 - stats[0])/stats[1]
return img, name<define_variables>
counts = Counter(FN_text.split())
 print(counts )
EXP = -1.2
G_logits = []
V_logits = []
C_logits = []
G_preds = []
V_preds=[]
C_preds = []<import_modules>
counts = Counter(FP_text.split())
 print(counts )
import scipy<statistical_test>
df_test['prediction']=(y_pred_test_final >= model_cutoff ).astype(int )
row_id,target = [],[]
for fname in TEST:
ds = GraphemeDataset(fname)
 dl = DataLoader(ds, batch_size=bs, num_workers=nworkers, shuffle=False)
 with torch.no_grad() :
for x,y in tqdm(dl):
x = x.unsqueeze(1 ).cuda()
p1,p2,p3 = model(x)
 p1 = torch.nn.functional.softmax(p1)
 p2 = torch.nn.functional.softmax(p2)
 p3 = torch.nn.functional.softmax(p3)
 G_logits.append(p1.cpu())
 V_logits.append(p2.cpu())
 C_logits.append(p3.cpu())
 p1 = p1.argmax(-1 ).view(-1 ).cpu()
p2 = p2.argmax(-1 ).view(-1 ).cpu()
p3 = p3.argmax(-1 ).view(-1 ).cpu()
G_preds.append(p1)
 V_preds.append(p2)
 C_preds.append(p3)
<concatenate>
df_test=df_test.drop(['question_text'], axis=1)
 df_test=df_test.drop(['length'], axis=1 )
<data_type_conversions><EOS> 
df_test.to_csv(r'submission.csv', index = False )