Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<compute_train_metric>
tqdm.pandas()
warnings.simplefilter(action='ignore', category=FutureWarning )
score = f1_score(
 cm_correct_labels,
cm_predictions,
labels=labels,
average='macro', 
)
 precision = precision_score(
 cm_correct_labels,
cm_predictions,
labels=labels,
average='macro', 
)
 recall = recall_score(
 cm_correct_labels,
cm_predictions,
labels=labels,
average='macro', 
)
 display_confusion_matrix(cmat, score, precision, recall )<define_variables>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
tf.set_random_seed(seed )
dataset = get_validation_dataset()
dataset = dataset.unbatch().batch(20)
 batch = iter(dataset )<predict_on_test>
seed = 5583
seed_everything(seed)
 begin = time.time()
sns.set_style('whitegrid' )
images, labels = next(batch)
 probabilities = model.predict(images)
 predictions = np.argmax(probabilities, axis=-1)
 display_batch_of_images(( images, labels), predictions )<predict_on_test>
GLOVE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
PARAGRAM = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
FASTTEXT = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
test_ds = get_test_dataset(ordered=True)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = model.predict(test_images_ds)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions )<save_to_csv>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt(
 'submission.csv',
np.rec.fromarrays([test_ids, predictions]),
fmt=['%s', '%d'],
delimiter=',',
header='id,label',
comments='', 
)
 !head submission.csv<set_options>
augment_insincere_count = 0
os.environ["WANDB_API_KEY"] = "0"
np.random.seed(0 )<load_from_csv>
class MarkovifyTextAugmentor(object):
def __init__(self, docs, gen_max_len, filter_min_len=10):

 self.mk_text_model = mk.Text(docs)
 self.gen_max_len = gen_max_len
self.filter_min_len = filter_min_len
def genenerate(self, count):
texts = []
for _ in range(count):
text = self.mk_text_model.make_short_sentence(self.gen_max_len)
 if text and len(text.split(' ')) > self.filter_min_len:
texts.append(text)
 return texts
train = pd.read_csv(".. /input/contradictory-my-dear-watson/train.csv")
 test = pd.read_csv(".. /input/contradictory-my-dear-watson/test.csv")
 train.head()<count_missing_values>
insincere_df = train_df[train_df['target'] == 1]
insincere_texts = insincere_df['question_text'].apply(" ".join)
 median_length = int(insincere_texts.str.len().median())
 mk_augmentor = MarkovifyTextAugmentor(insincere_texts, median_length, filter_min_len=6 )
missing_values_count = train.isnull().sum()
print("Number of missing data points per column:
")
 print(missing_values_count )<train_model>
%%time
def gen_text_process(count):
return mk_augmentor.genenerate(count)
 num_cores = psutil.cpu_count()
pool = multiprocessing.Pool(processes=num_cores)
 apply_results = []
process_counts = augment_insincere_count // num_cores
for _ in range(num_cores):
apply_results.append(pool.apply_async(gen_text_process,(process_counts,)))
 pool.close()
pool.join()
generated_texts = []
for res in apply_results:
generated_texts.extend(res.get())
 gen_df = pd.DataFrame({'question_text': generated_texts, 'target': [1] * len(generated_texts)} )
print("Number of training data rows: {}
".format(train.shape[0]))
print("Number of training data columns: {}
".format(train.shape[1]))<count_values>
print('Generated {} insincere questions'.format(len(generated_texts)) )
train.language.unique()
train.language.value_counts()<split>
if augment_insincere_count > 0:
train_df = pd.concat(( train_df, gen_df), axis=0, sort=True)
 train_df = shuffle(train_df, random_state=seed )
train, validation = train_test_split(train, stratify=train.label.values,
random_state=42,
test_size=0.2, shuffle=True )<train_model>
toxic_words = ['4r5e', '5h1t', '5hit', 'a55', 'anal', 'anus', 'ar5e', 'arrse', 'arse', 'ass', 'ass-fucker', 'asses', 'assfucker', 'assfukka', 'asshole', 'assholes', 'asswhole', 'a_s_s', 'b!tch', 'b00bs', 'b17ch', 'b1tch', 'ballbag', 'balls', 'ballsack', 'bastard', 'beastial', 'beastiality', 'bellend', 'bestial', 'bestiality', 'bi+ch', 'biatch', 'bitch', 'bitcher', 'bitchers', 'bitches', 'bitchin', 'bitching', 'bloody', 'blow', 'job', 'blowjob', 'blowjobs', 'boiolas', 'bollock', 'bollok', 'boner', 'boob', 'boobs', 'booobs', 'boooobs', 'booooobs', 'booooooobs', 'breasts', 'buceta', 'bugger', 'bum', 'bunny', 'fucker', 'butt', 'butthole', 'buttmuch', 'buttplug', 'c0ck', 'c0cksucker', 'carpet', 'muncher', 'cawk', 'chink', 'cipa', 'cl1t', 'clit', 'clitoris', 'clits', 'cnut', 'cock', 'cock-sucker', 'cockface', 'cockhead', 'cockmunch', 'cockmuncher', 'cocks', 'cocksuck', 'cocksucked', 'cocksucker', 'cocksucking', 'cocksucks', 'cocksuka', 'cocksukka', 'cok', 'cokmuncher', 'coksucka', 'coon', 'cox', 'crap', 'cum', 'cummer', 'cumming', 'cums', 'cumshot', 'cunilingus', 'cunillingus', 'cunnilingus', 'cunt', 'cuntlick', 'cuntlicker', 'cuntlicking', 'cunts', 'cyalis', 'cyberfuc', 'cyberfuck', 'cyberfucked', 'cyberfucker', 'cyberfuckers', 'cyberfucking', 'd1ck', 'damn', 'dick', 'dickhead', 'dildo', 'dildos', 'dink', 'dinks', 'dirsa', 'dlck', 'dog-fucker', 'doggin', 'dogging', 'donkeyribber', 'doosh', 'duche', 'dyke', 'ejaculate', 'ejaculated', 'ejaculates', 'ejaculating', 'ejaculatings', 'ejaculation', 'ejakulate', 'f', 'u', 'c', 'k', 'f', 'u', 'c', 'k', 'e', 'r', 'f4nny', 'fag', 'fagging', 'faggitt', 'faggot', 'faggs', 'fagot', 'fagots', 'fags', 'fanny', 'fannyflaps', 'fannyfucker', 'fanyy', 'fatass', 'fcuk', 'fcuker', 'fcuking', 'feck', 'fecker', 'felching', 'fellate', 'fellatio', 'fingerfuck', 'fingerfucked', 'fingerfucker', 'fingerfuckers', 'fingerfucking', 'fingerfucks', 'fistfuck', 'fistfucked', 'fistfucker', 'fistfuckers', 'fistfucking', 'fistfuckings', 'fistfucks', 'flange', 'fook', 'fooker', 'fuck', 'fucka', 'fucked', 'fucker', 'fuckers', 'fuckhead', 'fuckheads', 'fuckin', 'fucking', 'fuckings', 'fuckingshitmotherfucker', 'fuckme', 'fucks', 'fuckwhit', 'fuckwit', 'fudge', 'packer', 'fudgepacker', 'fuk', 'fuker', 'fukker', 'fukkin', 'fuks', 'fukwhit', 'fukwit', 'fux', 'fux0r', 'f_u_c_k', 'gangbang', 'gangbanged', 'gangbangs', 'gaylord', 'gaysex', 'goatse', 'God', 'god-dam', 'god-damned', 'goddamn', 'goddamned', 'hardcoresex', 'hell', 'heshe', 'hoar', 'hoare', 'hoer', 'homo', 'hore', 'horniest', 'horny', 'hotsex', 'jack-off', 'jackoff', 'jap', 'jerk-off', 'jism', 'jiz', 'jizm', 'jizz', 'kawk', 'knob', 'knobead', 'knobed', 'knobend', 'knobhead', 'knobjocky', 'knobjokey', 'kock', 'kondum', 'kondums', 'kum', 'kummer', 'kumming', 'kums', 'kunilingus', 'l3i+ch', 'l3itch', 'labia', 'lmfao', 'lust', 'lusting', 'm0f0', 'm0fo', 'm45terbate', 'ma5terb8', 'ma5terbate', 'masochist', 'master-bate', 'masterb8', 'masterbat*', 'masterbat3', 'masterbate', 'masterbation', 'masterbations', 'masturbate', 'mo-fo', 'mof0', 'mofo', 'mothafuck', 'mothafucka', 'mothafuckas', 'mothafuckaz', 'mothafucked', 'mothafucker', 'mothafuckers', 'mothafuckin', 'mothafucking', 'mothafuckings', 'mothafucks', 'mother', 'fucker', 'motherfuck', 'motherfucked', 'motherfucker', 'motherfuckers', 'motherfuckin', 'motherfucking', 'motherfuckings', 'motherfuckka', 'motherfucks', 'muff', 'mutha', 'muthafecker', 'muthafuckker', 'muther', 'mutherfucker', 'n1gga', 'n1gger', 'nazi', 'nigg3r', 'nigg4h', 'nigga', 'niggah', 'niggas', 'niggaz', 'nigger', 'niggers', 'nob', 'nob', 'jokey', 'nobhead', 'nobjocky', 'nobjokey', 'numbnuts', 'nutsack', 'orgasim', 'orgasims', 'orgasm', 'orgasms', 'p0rn', 'pawn', 'pecker', 'penis', 'penisfucker', 'phonesex', 'phuck', 'phuk', 'phuked', 'phuking', 'phukked', 'phukking', 'phuks', 'phuq', 'pigfucker', 'pimpis', 'piss', 'pissed', 'pisser', 'pissers', 'pisses', 'pissflaps', 'pissin', 'pissing', 'pissoff', 'poop', 'porn', 'porno', 'pornography', 'pornos', 'prick', 'pricks', 'pron', 'pube', 'pusse', 'pussi', 'pussies', 'pussy', 'pussys', 'rectum', 'retard', 'rimjaw', 'rimming', 's', 'hit', 's.o.b.', 'sadist', 'schlong', 'screwing', 'scroat', 'scrote', 'scrotum', 'semen', 'sex', 'sh!+', 'sh!t', 'sh1t', 'shag', 'shagger', 'shaggin', 'shagging', 'shemale', 'shi+', 'shit', 'shitdick', 'shite', 'shited', 'shitey', 'shitfuck', 'shitfull', 'shithead', 'shiting', 'shitings', 'shits', 'shitted', 'shitter', 'shitters', 'shitting', 'shittings', 'shitty', 'skank', 'slut', 'sluts', 'smegma', 'smut', 'snatch', 'son-of-a-bitch', 'spac', 'spunk', 's_h_i_t', 't1tt1e5', 't1tties', 'teets', 'teez', 'testical', 'testicle', 'tit', 'titfuck', 'tits', 'titt', 'tittie5', 'tittiefucker', 'titties', 'tittyfuck', 'tittywank', 'titwank', 'tosser', 'turd', 'tw4t', 'twat', 'twathead', 'twatty', 'twunt', 'twunter', 'v14gra', 'v1gra', 'vagina', 'viagra', 'vulva', 'w00se', 'wang', 'wank', 'wanker', 'wanky', 'whoar', 'whore', 'willies', 'willy', 'xrated', 'xxx']
print("Train data: {}
".format(train.shape))
print("Validation data: {}
".format(validation.shape))<load_pretrained>
def toxic_words_ratio(text, toxic_words=toxic_words):
count = 0
text = word_tokenize(text)
 for word in text:
count += int(word.lower() in toxic_words)
 return count / len(text )
model_name = 'bert-base-multilingual-cased'
tokenizer = BertTokenizer.from_pretrained(model_name)
 save_path = '.'
if not os.path.exists(save_path):
os.makedirs(save_path)
 tokenizer.save_pretrained(save_path)
 tokenizer = BertWordPieceTokenizer("vocab.txt", lowercase=False, strip_accents=False)
 tokenizer<categorify>
def build_vocab(texts):
vocab = {}
for sentence in texts:
for word in sentence.split(' '):
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
tokenized_premise = tokenizer.encode_batch(train.premise.values.tolist())
 train['premise_seq_length'] = [len(encoding.tokens)for encoding in tokenized_premise]
tokenized_hypothesis = tokenizer.encode_batch(train.hypothesis.values.tolist())
 train['hypothesis_seq_length'] = [len(encoding.tokens)for encoding in tokenized_hypothesis]
info_per_lang = train.groupby('language' ).agg({'premise_seq_length': ['mean', 'max', 'count'], 'hypothesis_seq_length': ['mean', 'max', 'count']})
 print(info_per_lang )<define_variables>
word_count = build_vocab(list(train_df['question_text'])+ list(test_df['question_text']))
EPOCHS = 3
BATCH_SIZE = 64
MAX_LEN = 100
PATIENCE = 1
LEARNING_RATE = 1e-5<categorify>
def freq_count(text):
text = text.split(" ")
 all_count = 0
for word in text:
all_count += word_count[word]
return len(text)/ all_count
def encode(df, tokenizer, max_len=50):
pairs = df[['premise','hypothesis']].values.tolist()
tokenizer.enable_truncation(max_len)
 tokenizer.enable_padding()
print("Encoding...")
 enc_list = tokenizer.encode_batch(pairs)
 print("Complete")
 input_word_ids = tf.ragged.constant([enc.ids for enc in enc_list], dtype=tf.int32)
 input_mask = tf.ragged.constant([enc.attention_mask for enc in enc_list], dtype=tf.int32)
 input_type_ids = tf.ragged.constant([enc.type_ids for enc in enc_list], dtype=tf.int32)
 inputs = {
'input_word_ids': input_word_ids.to_tensor() ,
'input_mask': input_mask.to_tensor() ,
'input_type_ids': input_type_ids.to_tensor() }
return inputs<categorify>
def make_features(df):
df['total_length'] = df['question_text'].apply(len)
 df['capitals'] = df['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']), axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
df['toxic_ratio'] = df['question_text'].apply(toxic_words_ratio)
 df['word_freq'] = df['question_text'].apply(freq_count )
train_input = encode(train, tokenizer=tokenizer, max_len=MAX_LEN )<categorify>
features = ['caps_vs_length', 'words_vs_unique', 'toxic_ratio', 'total_length', 'word_freq']
nb_features = len(features)
 print(f'Generated {nb_features} features :', ', '.join(features))
validation_input = encode(validation, tokenizer=tokenizer, max_len=MAX_LEN )<categorify>
%%time
make_features(train_df)
 make_features(test_df )
def build_model(model_name, max_len=50):
tf.random.set_seed(1234)
 bert_encoder = TFBertModel.from_pretrained(model_name)
 input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 input_type_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_type_ids")
 embedding = bert_encoder([input_word_ids, input_mask, input_type_ids])[0]
cls_vector = embedding[:,0,:]
output = tf.keras.layers.Dense(3, activation='softmax' )(cls_vector)
 model = tf.keras.Model(inputs=[input_word_ids, input_mask, input_type_ids], outputs=output)
 model.compile(tf.keras.optimizers.Adam(lr=LEARNING_RATE), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
 return model<train_model>
features_train = train_df[features].fillna(0)
 features_test = test_df[features].fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( features_train, features_test)))
 features_train = ss.transform(features_train)
 features_test = ss.transform(features_test )
checkpoint_filepath='bert_best_checkpoint.hdf5'
callbacks = [EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=PATIENCE), ModelCheckpoint(filepath=checkpoint_filepath, save_best_only=True, save_weights_only=True, monitor='val_accuracy', mode='max', verbose=1)]
train_history = model.fit(x=train_input, y=train.label.values, validation_data=(validation_input, validation.label.values), epochs=EPOCHS, verbose=1, batch_size=BATCH_SIZE, callbacks=callbacks )<drop_column>
useful_punct = ['_', '☹', '＞', '½', '△', '¿', '¼', '∆', '≥', '⇒', '¬', '∨', '＾', 'µ', '?', '!', '.', ',', '"', '
useless_punct = ['च', '不', 'ঢ়', '平', 'ᠠ', '錯', '判', '∙', '言', 'ς', 'ل', '្', 'ジ', 'あ', '得', '水', 'ь', '◦', '创', '康', '華', 'ḵ', '☺', '支', '就', '„', '」', '어', '谈', '陈', '团', '腻', '权', '年', '业', 'マ', 'य', 'ا', '売', '甲', '拼', '˂', 'ὤ', '贯', '亚', 'ि', '放', 'ʻ', 'ទ', 'ʖ', '點', '્', '発', '青', '能', '木', 'д', '微', '藤', '̃', '僕', '妒', '͜', 'ន', 'ध', '이', '希', '特', 'ड', '¢', '滢', 'ส', '나', '女', 'క', '没', '什', 'з', '天', '南', 'ʿ', 'ค', 'も', '凰', '步', '籍', '西', 'ำ', '−', 'л', 'ڤ', 'ៃ', '號', 'ص', 'स', '®', 'ʋ', '批', 'រ', '치', '谢', '生', '道', '═', '下', '俄', 'ɖ', '觀', 'வ', '—', 'ی', '您', '♥', '一', 'や', '⊆', 'ʌ', '語', 'ี', '兴', '惶', '瀛', '狐', '⁴', 'प', '臣', 'ద', '―', 'ì', 'ऌ', 'ీ', '自', '信', '健', '受', 'ɨ', '시', 'י', 'ছ', '嬛', '湾', '吃', 'ち', 'ड़', '反', '红', '有', '配', 'ে', 'ឯ', '宮', 'つ', 'μ', '記', '口', '℅ι', 'ो', '狸', '奇', 'о', 'ट', '聖', '蘭', '読', 'ū', '標', '要', 'ត', '识', 'で', '汤', 'ま', 'ʀ', '局', 'リ', '्', 'ไ', '呢', '工', 'ल', '沒', 'τ', 'ិ', 'ö', 'せ', '你', 'ん', 'ュ', '枚', '部', '大', '罗', 'হ', 'て', '表', '报', '攻', 'ĺ', 'ฉ', '∩', '宝', '对', '字', '文', '这', '∑', '髪', 'り', '่', '능', '罢', '내', '阻', '为', '菲', 'ي', 'न', 'ί', 'ɦ', '開', '†', '茹', '做', '東', 'ত', 'に', 'ت', '晓', '키', '悲', 'સ', '好', '›', '上', '存', '없', '하', '知', 'ធ', '斯', ' ', '授', 'ł', '傳', '兰', '封', 'ோ', 'و', 'х', 'だ', '人', '太', '品', '毒', 'ᡳ', '血', '席', '剔', 'п', '蛋', '王', '那', '梦', 'ី', '彩', '甄', 'и', '柏', 'ਨ', '和', '坊', '⌚', '广', '依', '∫', 'į', '故', 'ś', 'ऊ', '几', '日', 'ک', '音', '×', '”', '▾', 'ʊ', 'ज', 'ด', 'ठ', 'उ', 'る', '清', 'ग', 'ط', 'δ', 'ʏ', '官', '∛', '়', '้', '男', '骂', '复', '∂', 'ー', '过', 'য', '以', '短', '翻', 'র', '教', '儀', 'ɛ', '‹', 'へ', '¾', '合', '学', 'ٌ', '학', '挑', 'ष', '比', '体', 'م', 'س', 'អ', 'ת', '訓', '∀', '迎', 'វ', 'ɔ', '٨', '▒', '化', 'చ', '‛', 'প', 'º', 'น', '업', '说', 'ご', '¸', '₹', '儿', '︠', '게', '骨', 'ท', 'ऋ', 'ホ', '茶', '는', 'જ', 'ุ', '羡', '節', 'ਮ', 'উ', '番', 'ড়', '讲', 'ㅜ', '등', '伟', 'จ', '我', 'ล', 'す', 'い', 'ញ', '看', 'ċ', '∧', 'भ', 'ઘ', 'ั', 'ម', '街', 'ય', '还', '鰹', 'ខ', 'ు', '訊', 'म', 'ю', '復', '杨', 'ق', 'त', '金', '味', 'ব', '风', '意', '몇', '佬', '爾', '精', '¶', 'ం', '乱', 'χ', '교', 'ה', '始', 'ᠰ', '了', '个', '克', '্', 'ห', '已', 'ʃ', 'わ', '新', '译', '︡', '本', 'ง', 'б', 'け', 'ి', '明', '¯', '過', 'ك', 'ῥ', 'ف', 'ß', '서', '进', 'ដ', '样', '乐', '寧', '€', 'ณ', 'ル', '乡', '子', 'ﬁ', 'ج', '慕', '–', 'ᡵ', 'Ø', '͡', '제', 'Ω', 'ប', '絕', '눈', 'फ', 'ম', 'గ', '他', 'α', 'ξ', '§', 'ஜ', '黎', 'ね', '복', 'π', 'ú', '鸡', '话', '会', 'ক', '八', '之', '북', 'ن', '¦', '가', 'ו', '恋', '地', 'ῆ', '許', '产', 'ॡ', 'ش', '़', '野', 'ή', 'ɒ', '啧', 'យ', '᠌', 'ᠨ', 'ب', '皎', '老', '公', '☆', 'व', 'ি', 'ល', 'ر', 'គ', '행', 'ង', 'ο', '让', 'ំ', 'λ', 'خ', 'ἰ', '家', 'ট', 'ब', '理', '是', 'め', 'र', '√', '기', 'ν', '玉', '한', '入', 'ד', '别', 'د', 'ะ', '电', 'ા', '♫', 'ع', 'ં', '堵', '嫉', '伊', 'う', '千', '관', '篇', 'क', '非', '荣', '粵', '瑜', '英', '를', '美', '条', '`', '宋', '←', '수', '後', '•', '³', 'ी', '고', '肉', '℃', 'し', '漢', '싱', 'ϵ', '送', 'ه', '落', 'న', 'ក', 'க', 'ℇ', 'た', 'ះ', '中', '射', '♪', '符', 'ឃ', '谷', '分', '酱', 'び', 'থ', 'ة', 'г', 'σ', 'と', '楚', '胡', '饭', 'み', '禮', '主', '直', '÷', '夢', 'ɾ', 'চ', '⃗', '統', '高', '顺', '据', 'ら', '頭', 'よ', '最', 'ా', 'ੁ', '亲', 'ស', '花', '≡', '眼', '病', '…', 'の', '發', 'ா', '汝', '★', '氏', 'ร', '景', 'ᡠ', '读', '件', '仲', 'শ', 'お', 'っ', 'پ', 'ᡤ', 'ч', '♭', '悠', 'ं', '六', '也', 'ռ', 'য়', '恐', 'ह', '可', '啊', '莫', '书', '总', 'ষ', 'ք', '̂', '간', 'な', '此', '愛', 'ర', 'ใ', '陳', 'Ἀ', 'ण', '望', 'द', '请', '油', '露', '니', 'ş', '宗', 'ʍ', '鳳', 'अ', '邋', '的', 'ព', '火', 'ा', 'ก', '約', 'ட', '章', '長', '商', '台', '勢', 'さ', '국', 'Î', '簡', 'ई', '∈', 'ṭ', '經', '族', 'ु', '孫', '身', '坑', 'স', '么', 'ε', '失', '殺', 'ž', 'ર', 'が', '手', 'ា', '心', 'ਾ', '로', '朝', '们', '黒', '欢', '早', '️', 'া', 'आ', 'ɸ', '常', '快', '民', 'ﷺ', 'ូ', '遢', 'η', '国', '无', '江', 'ॠ', '「', 'ন', '™', 'ើ', 'ζ', '紫', 'ె', 'я', '“', '♨', '國', 'े', 'อ', '∞']
letter_mapping = {'\u200b':' ', 'ũ': "u", 'ẽ': 'e', 'é': "e", 'á': "a", 'ķ': 'k', 'ï': 'i', 'Ź': 'Z', 'Ż': 'Z', 'Š': 'S', 'Π': ' pi ', 'Ö': 'O', 'É': 'E', 'Ñ': 'N', 'Ž': 'Z', 'ệ': 'e', '²': '2', 'Å': 'A', 'Ā': 'A', 'ế': 'e', 'ễ': 'e', 'ộ': 'o', '⧼': '<', '⧽': '>', 'Ü': 'U', 'Δ': 'delta', 'ợ': 'o', 'İ': 'I', 'Я': 'R', 'О': 'O', 'Č': 'C', 'П': 'pi', 'В': 'B', 'Φ': 'phi', 'ỵ': 'y', 'օ': 'o', 'Ľ': 'L', 'ả': 'a', 'Γ': 'theta', 'Ó': 'O', 'Í': 'I', 'ấ': 'a', 'ụ': 'u', 'Ō': 'O', 'Ο': 'O', 'Σ': 'sigma', 'Â': 'A', 'Ã': 'A', 'ᗯ': 'w', 'ᕼ': "h", "ᗩ": "a", "ᖇ": "r", "ᗯ": "w", "O": "o", "ᗰ": "m", "ᑎ": "n", "ᐯ": "v", "н": "h", "м": "m", "o": "o", "т": "t", "в": "b", "υ": "u", "ι": "i","н": "h", "č": "c", "š": "s", "ḥ": "h", "ā": "a", "ī": "i", "à": "a", "ý": "y", "ò": "o", "è": "e", "ù": "u", "â": "a", "ğ": "g", "ó": "o", "ê": "e", "ạ": "a", "ü": "u", "ä": "a", "í": "i", "ō": "o", "ñ": "n", "ç": "c", "ã": "a", "ć": "c", "ô": "o", "с": "c", "ě": "e", "æ": "ae", "î": "i", "ő": "o", "å": "a", "Ä": "A", }
del model<drop_column>
mispell_dict = {"trimp": "trump", "wanket": "wanker",'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'qoura': 'quora', 'sallary': 'salary', 'whta': 'what', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'etherium': 'ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization',}
K.clear_session()<load_pretrained>
def add_maj(dic):
d = dic.copy()
for w in dic.keys() :
try:
d[w[0].upper() + w[1:]] = dic[w][0].upper() + dic[w][1:]
except:
d[w[0].upper() ] = dic[w][0].upper()
return d
PRETRAINED_MODEL_TYPES = {
'xlmroberta':(XLMRobertaConfig, TFXLMRobertaModel, XLMRobertaTokenizer, 'jplu/tf-xlm-roberta-large')
 }
config_class, model_class, tokenizer_class, model_name = PRETRAINED_MODEL_TYPES['xlmroberta']
tokenizer = AutoTokenizer.from_pretrained(model_name)
 tokenizer<categorify>
substitution_dic = {}
substitution_dic.update(mispell_dict)
 substitution_dic = add_maj(substitution_dic)
 substitution_dic.update(letter_mapping )
def encode(df, tokenizer, max_len=50):
pairs = df[['premise','hypothesis']].values.tolist()
print("Encoding...")
 encoded_dict = tokenizer.batch_encode_plus(pairs, max_length=max_len, padding=True, truncation=True,
add_special_tokens=True, return_attention_mask=True)
 print("Complete")
 input_word_ids = tf.convert_to_tensor(encoded_dict['input_ids'], dtype=tf.int32)
 input_mask = tf.convert_to_tensor(encoded_dict['attention_mask'], dtype=tf.int32)
 inputs = {
'input_word_ids': input_word_ids,
'input_mask': input_mask}
return inputs<categorify>
def _get_substitution(sub_dic):
sub_re = re.compile('(%s)' % '|'.join(sub_dic.keys()))
return sub_dic, sub_re
substitutions, substitutions_re = _get_substitution(substitution_dic)
 def replace_substitution(text):
def replace(match):
return substitutions[match.group(0)]
return substitutions_re.sub(replace, text )
train_input = encode(train, tokenizer=tokenizer, max_len=MAX_LEN )<categorify>
def clean_apostrophes(x):
apostrophes = ["’", "‘", "´", "`"]
for s in apostrophes:
x = re.sub(s, "'", x)
 return x
validation_input = encode(validation, tokenizer=tokenizer, max_len=MAX_LEN )<categorify>
def remove_s(x):
if len(x)> 2:
return re.sub("('$ |'$|'s |'s)", ' ', x)
 else:
return x
def build_model(max_len=50):
tf.random.set_seed(1234)
 encoder = model_class.from_pretrained(model_name)
 input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 embedding = encoder([input_word_ids, input_mask])[0]
sequence_output = embedding[:,0,:]
output = tf.keras.layers.Dense(3, activation="softmax" )(sequence_output)
 model = tf.keras.Model(inputs=[input_word_ids, input_mask], outputs=output)
 model.compile(tf.keras.optimizers.Adam(lr=LEARNING_RATE), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
 return model<train_model>
spaces = ['\u200b', '\u200e', '\u202a', '\u202c', '\ufeff', '\uf0d8', '\u2061', '\x10', '\x7f', '\x9d', '\xad', '\xa0']
def clean_spaces(text):
for space in spaces:
text = text.replace(space, ' ')
 text = text.strip()
text = re.sub('\s+', ' ', text)
 return text
checkpoint_filepath='xlmroberta_best_checkpoint.hdf5'
callbacks = [EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=PATIENCE), ModelCheckpoint(filepath=checkpoint_filepath, save_best_only=True, save_weights_only=True, monitor='val_accuracy', mode='max', verbose=1)]
train_history = model.fit(x=train_input, y=train.label.values, validation_data=(validation_input, validation.label.values), epochs=EPOCHS, verbose=1, batch_size=BATCH_SIZE, callbacks=callbacks )<feature_engineering>
def decontract(text):
text = re.sub(r"(W|w)on('|\’)t ", "will not ", text)
 text = re.sub(r"(C|c)an('|\’)t ", "can not ", text)
 text = re.sub(r"(Y|y )('|\’)all ", "you all ", text)
 text = re.sub(r"(Y|y)a('|\’)ll ", "you all ", text)
 text = re.sub(r"(I|i )('|\’)m ", "i am ", text)
 text = re.sub(r"(A|a)isn('|\’)t ", "is not ", text)
 text = re.sub(r"n('|\’)t ", " not ", text)
 text = re.sub(r"('|\’)re ", " are ", text)
 text = re.sub(r"('|\’)s ", " is ", text)
 text = re.sub(r"('|\’)d ", " would ", text)
 text = re.sub(r"('|\’)ll ", " will ", text)
 text = re.sub(r"('|\’)t ", " not ", text)
 text = re.sub(r"('|\’)ve ", " have ", text)
 return text
def accuracy(x):
return round(float(x[2]/x[1]), 2)*100
validation['predictions'] = validation_predictions
lang_counts = validation.language.value_counts().sort_index()
tp_per_lang = validation[validation['label'] == validation['predictions']].groupby('language' ).agg({'language': ['count']} ).sort_index()
lang_names = lang_counts.index.tolist()
lang_tuples = list(zip(lang_names, lang_counts.values.tolist() , tp_per_lang.iloc[:, 0].values.tolist()))
acc = map(accuracy, lang_tuples)
 for i, score in enumerate(acc):
print("Accuracy of {} is {} ".format(lang_tuples[i][0], score))<load_pretrained>
def clean_numbers(text):
text = re.sub(r'(\d+ )([a-zA-Z])', '\g<1> \g<2>', text)
 text = re.sub(r'(\d+ )(th|st|nd|rd)', '\g<1>\g<2> ', text)
 text = re.sub(r'(\d+),(\d+)', '\g<1>\g<2>', text)
 return text
model.load_weights(checkpoint_filepath )<categorify>
def clean_special_chars(text, punct=useful_punct):
for p in punct:
text = text.replace(p, f' {p} ')
 return text
test_input = encode(test, tokenizer=tokenizer, max_len=MAX_LEN )<predict_on_test>
def clean_latex_tag(text):
corr_t = []
for t in text.split(" "):
t = t.strip()
if t != '':
corr_t.append(t)
 text = ' '.join(corr_t)
 text = re.sub('(\[ math \] ).+(\[ / math \])', 'mathematical formula', text)
 return text
predictions = [np.argmax(i)for i in model.predict(test_input)] 
<prepare_output>
def build_vocab(texts):
vocab = {}
for sentence in texts:
for word in sentence.split(' '):
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
submission = test.id.copy().to_frame()
submission['prediction'] = predictions
submission.head()<save_to_csv>
print("Number of available cpu cores: {}".format(cpu_count()))
def process_in_parallel(function, list_):
with Pool(cpu_count())as p:
tmp = p.map(function, list_)
 return tmp
submission.to_csv("submission.csv", index = False )<install_modules>
def treat_texts(texts):
texts = process_in_parallel(clean_apostrophes, texts)
 texts = process_in_parallel(replace_substitution, texts)
 texts = process_in_parallel(decontract, texts)
 texts = process_in_parallel(remove_s, texts)
 texts = process_in_parallel(clean_numbers, texts)
 texts = process_in_parallel(clean_special_chars, texts)
 texts = process_in_parallel(clean_spaces, texts)
 texts = process_in_parallel(clean_latex_tag, texts)
 return texts
!pip install --upgrade pip
!pip install --upgrade allennlp
!pip install transformers==4.0.1<load_from_url>
%%time
train_df["question_text"] = treat_texts(train_df["question_text"])
 test_df["question_text"] = treat_texts(test_df["question_text"] )
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev<import_modules>
vocab = build_vocab(list(train_df["question_text"])+ list(test_df["question_text"]))
import transformers
import pandas as pd
import torch<import_modules>
len_voc = None
max_len = 70
import torch_xla
import torch_xla.core.xla_model as xm<import_modules>
def make_input_data(X_train, X_test):
t = Tokenizer(num_words=len_voc, filters='', lower=False)
 t.fit_on_texts(( np.concatenate(( X_train, X_test), axis=0)))
 X_train = t.texts_to_sequences(X_train)
 X_test = t.texts_to_sequences(X_test)
 X_train = pad_sequences(X_train, maxlen=max_len, padding='post', truncating='post')
 X_test = pad_sequences(X_test, maxlen=max_len, padding='post', truncating='post')
 return X_train, X_test, t.word_index
print('Transformers version: ', transformers.__version__)
 print('Pytorch version: ', torch.__version__ )<load_from_csv>
%%time
X_train, X_test, word_index = make_input_data(train_df['question_text'], test_df['question_text'] )
data_dir = '/kaggle/input/contradictory-my-dear-watson/'
train_df = pd.read_csv(data_dir+'train.csv' ).sample(frac=1, random_state=100)
 test_df = pd.read_csv(data_dir+'test.csv')
 print(train_df['label'].value_counts())
 train_df.head(5 )<load_pretrained>
len_voc = len(word_index)+ 1
PRE_TRAINED_MODEL = 'xlm-roberta-large'
tokenizer = transformers.AutoTokenizer.from_pretrained(PRE_TRAINED_MODEL )<define_variables>
y_train = train_df['target'].values
MAX_LEN = 84
split_idx = int(train_df.shape[0] * 0.8 )<categorify>
def make_embed_index(file, word_index, vocab=[], exceptions={}):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 if file == FASTTEXT:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file)if len(o)>100 and o.split(" ")[0] in word_index)
 if file == PARAGRAM:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding="utf8", errors='ignore')if len(o)>100 and o.split(" ")[0] in word_index)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file)if o.split(" ")[0] in word_index)
 for word in vocab:
try:
_ = embeddings_index[word]
except:
try:
embeddings_index[word] = embeddings_index[word.lower() ]
except:
try:
embeddings_index[word] = embeddings_index[word.upper() ]
except:
try:
embeddings_index[word] = embeddings_index[word[0].upper() + word[1:].lower() ]
except:
pass
return embeddings_index
def get_encode(data):
tokenized_data=tokenizer(
 text=list(data['premise']), text_pair=list(data['hypothesis']),
max_length=MAX_LEN,
pad_to_max_length=True,
add_special_tokens=True,
truncation=True,
return_attention_mask=True,
return_token_type_ids=True,
return_tensors='pt')
 return tokenized_data 
<string_transform>
def make_embed_mat(embeddings_index):
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(len(word_index)+1, embed_size))
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
seed=2
tokenized_train=get_encode(train_df[:split_idx])
 labels_train=torch.tensor(train_df.label.values[:split_idx])
 tokenized_valid=get_encode(train_df[split_idx:])
 labels_valid=torch.tensor(train_df.label.values[split_idx:])
 tokenized_test=get_encode(test_df )<create_dataframe>
%%time
glove_index = make_embed_index(GLOVE, word_index, vocab )
batch_size=64
train_data=TensorDataset(torch.tensor(tokenized_train['input_ids']),
torch.tensor(tokenized_train['token_type_ids']),torch.tensor(tokenized_train['attention_mask'])
 ,labels_train)
 train_sampler=RandomSampler(train_data)
 train_dataloader=DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 valid_data=TensorDataset(torch.tensor(tokenized_valid['input_ids']),
torch.tensor(tokenized_valid['token_type_ids']),
torch.tensor(tokenized_valid['attention_mask'])
 ,labels_valid)
 valid_sampler=SequentialSampler(valid_data)
 valid_dataloader=DataLoader(valid_data, sampler=valid_sampler, batch_size=batch_size)
 test_data=TensorDataset(torch.tensor(tokenized_test['input_ids']),
torch.tensor(tokenized_test['token_type_ids']),
torch.tensor(tokenized_test['attention_mask']))
test_dataloader=DataLoader(test_data, batch_size=batch_size )<load_pretrained>
%%time
fasttext_index = make_embed_index(FASTTEXT, word_index, vocab )
model = AutoModelForSequenceClassification.from_pretrained(PRE_TRAINED_MODEL,
num_labels = 3,
output_attentions = False,
output_hidden_states = False)
<set_options>
%%time
para_index = make_embed_index(PARAGRAM, word_index, vocab )
device = xm.xla_device()
torch.set_default_tensor_type('torch.FloatTensor')
 print(model.to(device))
print('TPU available')
<compute_test_metric>
def check_coverage(vocab, embeddings_index, word_index=None):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
def accuracy(predictions, labels):
prediction_flat = np.argmax(predictions, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(prediction_flat == labels_flat)/ len(labels_flat )<init_hyperparams>
print("Glove : ")
 oov_glove = check_coverage(vocab, glove_index, word_index)
 print("Paragram : ")
 oov_paragram = check_coverage(vocab, para_index, word_index)
 print("FastText : ")
 oov_fasttext = check_coverage(vocab, fasttext_index, word_index )
optimizer = AdamW(model.parameters() ,
lr = 2e-5 
)
 epochs = 20
total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps )<train_model>
embed_concat = np.concatenate(( make_embed_mat(fasttext_index), make_embed_mat(glove_index), make_embed_mat(para_index)) , axis=1 )
random.seed(10)
 np.random.seed(10)
 torch.manual_seed(10)
 torch.cuda.manual_seed_all(10)
 losses = []
for i in range(0, epochs):
print('Epoch {:} of {:} Training...'.format(i+1, epochs))
total_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 10 == 0 and step != 0:
print('[{}] Batch {:>5,} of {:>5,}' 
.format(datetime.datetime.now().strftime('%H:%M:%S'), step, len(train_dataloader)))
 train_batch_input = batch[0].to(device)
 train_batch_input_types = batch[1].to(device)
 train_batch_mask = batch[2].to(device)
 train_batch_label = batch[3].to(device)
 model.zero_grad()
outputs = model(train_batch_input, token_type_ids = train_batch_input_types,
attention_mask = train_batch_mask, labels = train_batch_label)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 xm.optimizer_step(optimizer, barrier=True)
 scheduler.step()
average_train_loss = total_loss / len(train_dataloader)
 losses.append(average_train_loss)
 print('Training loss={:.2f}'.format(average_train_loss))
model.eval()
eval_accuracy = 0
eval_count = 0
for batch in valid_dataloader:
valid_batch_input = batch[0].to(device)
 valid_batch_input_types = batch[1].to(device)
 valid_batch_mask = batch[2].to(device)
 valid_batch_labels = batch[3].to(device)
 with torch.no_grad() :
outputs = model(valid_batch_input, token_type_ids = valid_batch_input_types,
attention_mask = valid_batch_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = valid_batch_labels.to('cpu' ).numpy()
batch_accuracy = accuracy(logits, label_ids)
 eval_accuracy += batch_accuracy
eval_count += 1
print('Validation accuracy={:.2f}'.format(eval_accuracy / eval_count))
print('')
 if i == 4:
break
print("Training done")
<predict_on_test>
t_init = time.time()
print(f"Initialized in {(t_init - begin)// 60} minutes" )
model.eval()
submissions = []
for batch in test_dataloader:
test_batch_input = batch[0].to(device)
 test_batch_input_types = batch[1].to(device)
 test_batch_mask = batch[2].to(device)
 with torch.no_grad() :
outputs = model(test_batch_input, token_type_ids = test_batch_input_types,
attention_mask = test_batch_mask)
 logits = outputs[0]
submissions.extend(np.argmax(logits.detach().cpu().numpy() , axis=1 ).flatten())
<save_to_csv>
class Noise(nn.Module):
def __init__(self, mean=0.0, stddev=0.1):
super(Noise, self ).__init__()
self.mean = mean
self.stddev = stddev
def forward(self, input):
noise = input.clone().normal_(self.mean, self.stddev)
 return input + noise
output = pd.DataFrame({'id': test_df.id,
'prediction': submissions})
 output.to_csv('submission.csv', index=False )<install_modules>
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(x.contiguous().view(-1, feature_dim), self.weight ).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )
!pip install datasets
!pip install --upgrade seaborn<import_modules>
class Model(nn.Module):
def __init__(self, embedding_matrix):
super(Model, self ).__init__()
h1 = 64
h2 = 32
hd = 32
self.embedding = nn.Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1])
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.noise = Noise(stddev=0.05)
 self.lstm = nn.LSTM(embedding_matrix.shape[1], 2*h1, bidirectional=False, batch_first=True)
 self.gru = nn.GRU(2*h1, 2*h2, bidirectional=False, batch_first=True)
 self.lstm_att = Attention(2*h1, max_len)
 self.gru_att = Attention(2*h2, max_len)
 self.dropout = nn.Dropout(0.1)
 self.linear = nn.Linear(6*(h1+h2)+ nb_features, hd)
 self.relu = nn.ReLU()
self.bn = nn.BatchNorm1d(hd, momentum=0.5)
 self.out = nn.Linear(hd, 1)
 def forward(self, x):
embed = self.embedding(x[0])
 if self.training:
embed = torch.squeeze(self.noise(torch.unsqueeze(embed, 0)))
 lstm, _ = self.lstm(embed)
 gru, _ = self.gru(lstm)
 att1 = self.lstm_att(lstm)
 att2 = self.gru_att(gru)
 avg_pool1 = torch.mean(lstm, 1)
 avg_pool2 = torch.mean(gru, 1)
 max_pool1, _ = torch.max(lstm, 1)
 max_pool2, _ = torch.max(gru, 1)
 conc = torch.cat(( att1, avg_pool1, max_pool1, att2, avg_pool2, max_pool2, x[1]), 1)
 conc = self.dropout(conc)
 conc = self.bn(self.relu(self.linear(conc)))
 conc = self.dropout(conc)
 out = self.out(conc)
 return out
import os
import seaborn as sns
import numpy as np
import random
from sklearn.utils import shuffle
import matplotlib.pyplot as plt
import pandas as pd
from collections import namedtuple
import plotly.express as px
import transformers
import tokenizers
import datetime
import time
from sklearn.metrics import accuracy_score, precision_recall_fscore_support, matthews_corrcoef
import tensorflow
from datasets import load_dataset
import torch
from torch import nn
from transformers import BertTokenizer, AutoTokenizer, TrainingArguments, AutoModel, AutoModelForSequenceClassification
from torch.utils.data import TensorDataset, random_split
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
from transformers import BertForSequenceClassification, AdamW, BertConfig
from transformers import get_linear_schedule_with_warmup
from transformers import RobertaTokenizer, RobertaForSequenceClassification, XLMRobertaModel<create_dataframe>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
mnli = load_dataset('glue', 'mnli')
 len(mnli )<create_dataframe>
def get_lr(epoch):
if epoch <= 3:
return 0.001
else:
return 0.0005
xnli = load_dataset('xnli', 'all_languages')
 len(xnli )<load_from_csv>
def tweak_threshold(train_preds, y_train):
tmp = [0,0,0]
delta = 0
for tmp[0] in np.arange(0.1, 0.501, 0.01):
tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
return tmp[2], delta
original_train_df = pd.read_csv(".. /input/contradictory-my-dear-watson/train.csv")
 original_train_df = shuffle(original_train_df)
 print(original_train_df.shape)
 original_valid_df = original_train_df[:len(original_train_df)//2]
original_train_df = original_train_df[(len(original_train_df)//2):]
print(original_train_df.shape, original_valid_df.shape)
 original_train_df.head()<load_from_csv>
def predict(X, f,model, batch_size=1024):
y = np.array([])
 X = torch.tensor(X, dtype=torch.long ).cuda()
f = torch.tensor(f, dtype=torch.float32 ).cuda()
Xf = torch.utils.data.TensorDataset(X, f)
 loader = torch.utils.data.DataLoader(Xf, batch_size=batch_size, shuffle=False)
 for i,(x, f)in enumerate(loader):
y = np.concatenate(( y, sigmoid(model([x, f] ).detach().cpu().numpy())[:, 0]))
return y
original_test_df = pd.read_csv(".. /input/contradictory-my-dear-watson/test.csv")
 original_test_df.shape<load_pretrained>
def fit(model, X_train, f_train, y_train, X_val=None, f_val=None, y_val=None, epochs=5, batch_size=512):
history = {"loss":[], "val_loss": []}
best_loss = 10
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')
 optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0.001)
 X_train = torch.tensor(X_train, dtype=torch.long ).cuda()
f_train = torch.tensor(f_train, dtype=torch.float32 ).cuda()
y_train = torch.tensor(y_train[:, np.newaxis], dtype=torch.float32 ).cuda()
train = torch.utils.data.TensorDataset(X_train, f_train, y_train)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 X_val = torch.tensor(X_val, dtype=torch.long ).cuda()
f_val = torch.tensor(f_val, dtype=torch.float32 ).cuda()
y_val = torch.tensor(y_val[:, np.newaxis], dtype=torch.float32 ).cuda()
val = torch.utils.data.TensorDataset(X_val, f_val, y_val)
 val_loader = torch.utils.data.DataLoader(val, batch_size=batch_size, shuffle=False)
 for epoch in range(epochs):
model.train()
avg_loss = 0
start_time = time.time()
lr = get_lr(epoch)
 for param_group in optimizer.param_groups:
param_group['lr'] = lr
for x_batch, f_batch, y_batch in train_loader:
optimizer.zero_grad()
y_pred = model([x_batch, f_batch])
 loss = loss_fn(y_pred, y_batch)
 loss.backward()
optimizer.step()
avg_loss += loss.item() / X_train.shape[0]
model.eval()
avg_val_loss = 0.
 for i,(x_batch, f_batch, y_batch)in enumerate(val_loader):
y_pred = model([x_batch, f_batch] ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / X_val.shape[0]
history['loss'].append(avg_loss)
 history['val_loss'].append(avg_val_loss)
 elapsed_time = time.time() - start_time
print('Epoch {}/{} \t lr={} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(epoch + 1, epochs, lr, avg_loss, avg_val_loss, elapsed_time))
return history
bert_model_name = 'joeddav/xlm-roberta-large-xnli'
max_len = 128
tokenizer = AutoTokenizer.from_pretrained(bert_model_name )<categorify>
def k_fold(model_class, embedding_matrix, X, f, y, X_test, f_test, k=5, batch_size=512, epochs=5, seed=seed):
splits = list(StratifiedKFold(n_splits=k, shuffle=True, random_state=420 ).split(X, y))
pred_test = np.zeros(X_test.shape[0])
 pred_oof = np.zeros(y.shape[0])
 histories = []
for i,(train_idx, val_idx)in enumerate(splits):
print(f"------------- Fold {i+1} -------------
")
 seed_everything(seed + i)
 cp_path = f"{i}_weights.pth.tar"
start_time = time.time()
model = model_class(embedding_matrix)
 history = fit(model, X[train_idx], f[train_idx], y[train_idx], X[val_idx], f[val_idx], y[val_idx], epochs=epochs, batch_size=batch_size)
 histories.append(history)
 pred_oof[val_idx] = predict(X[val_idx], f[val_idx], model)
 pred_test += predict(X_test, f_test, model)/ k
score, threshold = tweak_threshold(pred_oof[val_idx], y[val_idx])
 print(f"
Scored {score :.4f} for threshold {threshold :.3f} on validation data")
 print(f"
Done in {(time.time() - start_time)/ 60 :.1f} minutes
")
 return pred_test, pred_oof, histories
def encode(premise, hypothesis, label):
encoded_dict = tokenizer(
 premise,
hypothesis,
add_special_tokens = True,
truncation=True,
max_length = max_len,
padding='max_length',
return_attention_mask = True,
return_tensors = 'pt', 
)
 for k in encoded_dict:
encoded_dict[k] = torch.squeeze(encoded_dict[k])
 encoded_dict['labels'] = label
return encoded_dict<create_dataframe>
pred_test, pred_oof, histories = k_fold(Model, embed_concat,
X_train, features_train, y_train, X_test, features_test,
k=5, batch_size=512, epochs=4 )
raw_ds_mapping = {
'original train':(_get_raw_datasets_from_dataframe, original_train_df, len(original_train_df)) ,
'original valid':(_get_raw_datasets_from_dataframe, original_valid_df, len(original_valid_df)) ,
'mnli train' :(_get_raw_datasets_from_nlp, mnli['train'], mnli['train'].num_rows),
'mnli valid 1' :(_get_raw_datasets_from_nlp, mnli['validation_matched'], mnli['validation_matched'].num_rows),
'mnli valid 2' :(_get_raw_datasets_from_nlp, mnli['validation_mismatched'], mnli['validation_mismatched'].num_rows),
'xnli valid' :(_get_raw_datasets_from_nlp, xnli['validation'], xnli['validation'].num_rows * 15),
'xnli test' :(_get_raw_datasets_from_nlp, xnli['test'], xnli['test'].num_rows * 15),
'original test' :(_get_raw_datasets_from_dataframe, original_test_df, len(original_test_df)) ,
}
def get_raw_dataset(ds_name):
fn, ds, nb_examples = raw_ds_mapping[ds_name]
for x in fn(ds):
yield x
return<define_variables>
score, threshold = tweak_threshold(pred_oof, y_train)
 print(f"Local CV : {score:.4f} for threshold {threshold:.3f}" )
train_ds_names = ['original train', 'mnli train', 'mnli valid 1', 'mnli valid 2', 'xnli valid', 'xnli test']
eval_ds_names = ['original valid']
test_ds_names = ['original test']<categorify>
label_test =(pred_test > threshold ).astype(int )
class MyIterableDataset(torch.utils.data.IterableDataset):
def __init__(self, ds_names, isLabeledDataset=False):
super(MyIterableDataset ).__init__()
self.ds_names = ds_names
self.length = self.getLen()
self.isLabeledDataset = isLabeledDataset
return
def getLen(self):
length = 0
for ds_name in self.ds_names:
fn, ds, nb_examples = raw_ds_mapping[ds_name]
length += nb_examples
return length
def __iter__(self):
for ds_name in self.ds_names:
fn, ds, nb_examples = raw_ds_mapping[ds_name]
for x in fn(ds):
if self.isLabeledDataset and x['labels'] == -1:
continue
yield x
return
def __len__(self):
return self.length<create_dataframe>
output = pd.DataFrame({"qid": test_df["qid"].values})
 output['prediction'] = label_test
output.to_csv("submission.csv", index=False )
<load_pretrained><EOS> 
print(f"Ended in {(time.time() - begin)/ 60 :.1f} minutes" )