Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<predict_on_test>
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer("english")
 lemmatizer = WordNetLemmatizer()
test_ds = get_test_dataset(ordered=True)
 print('Вычисляем предсказания...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = model.predict(test_images_ds)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print('Создание файла submission.csv...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<feature_engineering>
data = pd.read_csv('.. /input/quora-insincere-questions-classification/train.csv')
 print("Number of data points:", data.shape )
os.environ["WANDB_API_KEY"] = "0" 
<define_variables>
test_data = pd.read_csv('.. /input/quora-insincere-questions-classification/test.csv')
 print("Number of data points:", test_data.shape )
SEED = 42
EPOCHS = 10
MAX_LEN = 96
NUM_SPLITS = 3
LR = 3e-5
BATCH_SIZE = 16<feature_engineering>
stopword = stopwords.words('english' )
os.environ['PYTHONHASHSEED']=str(SEED)
 random.seed(SEED)
 np.random.seed(SEED)
 tf.random.set_seed(SEED )<load_from_csv>
data['q_len'] = data['question_text'].str.len()
data['q_words'] = data['question_text'].apply(lambda row: len(row.split(" ")))
 data['u_chars'] = data['question_text'].apply(lambda row: sum(1 for c in row if c.isupper()))
data['l_chars'] = data['question_text'].apply(lambda row: sum(1 for c in row if c.islower()))
data['n_stopwords'] = data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if word in stopword))
data['n_cap_words'] = data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if word.isupper()))
data['n_diff_words'] = data['question_text'].apply(lambda row: len(set(row.split(" "))))
data['avg_word_len'] = data['question_text'].apply(lambda row: sum(len(i)for i in row.split(" ")) /len(row.split(" ")))
 data['n_numerical_words'] = data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if word.isnumeric()))
data["nb_simley"] = data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if re.findall(r"^(:\(|:\)) +$", word)))
 data["nb_symbols"] = data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if re.findall(r"[@_!
data["nb_punct"] = data['question_text'].apply(lambda row: sum(1 for c in row if(c=="'" or c==';' or c=="/" or c=='.')))
 print(data.shape)
train = pd.read_csv(".. /input/contradictory-my-dear-watson-translated-dataset/train_augmented.csv", index_col=["id"])
 test = pd.read_csv(".. /input/contradictory-my-dear-watson/test.csv", index_col=["id"])
 test["label"] = -1<feature_engineering>
test_data['q_len'] = test_data['question_text'].str.len()
test_data['q_words'] = test_data['question_text'].apply(lambda row: len(row.split(" ")))
 test_data['u_chars'] = test_data['question_text'].apply(lambda row: sum(1 for c in row if c.isupper()))
test_data['l_chars'] = test_data['question_text'].apply(lambda row: sum(1 for c in row if c.islower()))
test_data['n_stopwords'] = test_data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if word in stopword))
test_data['n_cap_words'] = test_data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if word.isupper()))
test_data['n_diff_words'] = test_data['question_text'].apply(lambda row: len(set(row.split(" "))))
test_data['avg_word_len'] = test_data['question_text'].apply(lambda row: sum(len(i)for i in row.split(" ")) /len(row.split(" ")))
 test_data['n_numerical_words'] = test_data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if word.isnumeric()))
test_data["nb_simley"] = test_data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if re.findall(r"^(:\(|:\)) +$", word)))
 test_data["nb_symbols"] = test_data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if re.findall(r"[@_!
test_data["nb_punct"] = test_data['question_text'].apply(lambda row: sum(1 for c in row if(c=="'" or c==';' or c=="/" or c=='.')))
 print(test_data.shape)
df = pd.concat([train, test])
 df.loc[df["label"]!=-1, "type"] = "train"
df.loc[df["label"]==-1, "type"] = "test"<set_options>
puncts = [
',', '.', '"', ':', ')', '(', '-', '!', '?','|', ';', "'", '$', '&',
'/', '[', ']', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '→', '°', '€', '™', '›',
'♥', '←', '×', '§', '″', '′', 'Â', '█', 'à', '…', '“', '★', '”',
'–', '●', 'â', '►', '−', '¢', '¬', '░', '¶', '↑', '±', '▾',
'═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '⊕', '▼',
'▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',
'è', '¸', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»',
'，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø',
'¹', '≤', '‡', '₹', '´'
]
del df
gc.collect()<choose_model_class>
abbreviations = {
"ain't": "is not",
"aren't": "are not",
"can't": "cannot",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he would",
"he'll": "he will",
"he's": "he is",
"how'd": "how did",
"how'd'y": "how do you",
"how'll": "how will",
"how's": "how is",
"I'd": "I would",
"I'd've": "I would have",
"I'll": "I will",
"I'll've": "I will have",
"I'm": "I am",
"I've": "I have",
"i'd": "i would",
"i'd've": "i would have",
"i'll": "i will",
"i'll've": "i will have",
"i'm": "i am",
"i've": "i have",
"isn't": "is not",
"it'd": "it would",
"it'd've": "it would have",
"it'll": "it will",
"it'll've": "it will have",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"mightn't've": "might not have",
"must've": "must have",
"mustn't": "must not",
"mustn't've": "must not have",
"needn't": "need not",
"needn't've": "need not have",
"o'clock": "of the clock",
"oughtn't": "ought not",
"oughtn't've": "ought not have",
"shan't": "shall not",
"sha'n't": "shall not",
"shan't've": "shall not have",
"she'd": "she would",
"she'd've": "she would have",
"she'll": "she will",
"she'll've": "she will have",
"she's": "she is",
"should've": "should have",
"shouldn't": "should not",
"shouldn't've": "should not have",
"so've": "so have",
"so's": "so as",
"this's": "this is",
"that'd": "that would",
"that'd've": "that would have",
"that's": "that is",
"there'd": "there would",
"there'd've": "there would have",
"there's": "there is",
"here's": "here is",
"they'd": "they would",
"they'd've": "they would have",
"they'll": "they will",
"they'll've": "they will have",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd": "we would",
"we'd've": "we would have",
"we'll": "we will",
"we'll've": "we will have",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what will",
"what'll've": "what will have",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"when's": "when is",
"when've": "when have",
"where'd": "where did",
"where's": "where is",
"where've": "where have",
"who'll": "who will",
"who'll've": "who will have",
"who's": "who is",
"who've": "who have",
"why's": "why is",
"why've": "why have",
"will've": "will have",
"won't": "will not",
"won't've": "will not have",
"would've": "would have",
"wouldn't": "would not",
"wouldn't've": "would not have",
"y'all": "you all",
"y'all'd": "you all would",
"y'all'd've": "you all would have",
"y'all're": "you all are",
"y'all've": "you all have",
"you'd": "you would",
"you'd've": "you would have",
"you'll": "you will",
"you'll've": "you will have",
"you're": "you are",
"you've": "you have",
"who'd": "who would",
"who're": "who are",
"'re": " are",
"tryin'": "trying",
"doesn'": "does not",
'howdo': 'how do',
'whatare': 'what are',
'howcan': 'how can',
'howmuch': 'how much',
'howmany': 'how many',
'whydo': 'why do',
'doI': 'do I',
'theBest': 'the best',
'howdoes': 'how does',
}
strategy = tf.distribute.experimental.TPUStrategy(tpu )<define_variables>
spells = {
'colour': 'color',
'centre': 'center',
'favourite': 'favorite',
'travelling': 'traveling',
'counselling': 'counseling',
'theatre': 'theater',
'cancelled': 'canceled',
'labour': 'labor',
'organisation': 'organization',
'wwii': 'world war 2',
'citicise': 'criticize',
'youtu.be': 'youtube',
'youtu ': 'youtube ',
'qoura': 'quora',
'sallary': 'salary',
'Whta': 'what',
'whta': 'what',
'narcisist': 'narcissist',
'mastrubation': 'masturbation',
'mastrubate': 'masturbate',
"mastrubating": 'masturbating',
'pennis': 'penis',
'Etherium': 'ethereum',
'etherium': 'ethereum',
'narcissit': 'narcissist',
'bigdata': 'big data',
'2k17': '2017',
'2k18': '2018',
'2k19': '2020',
'qouta': 'quota',
'exboyfriend': 'ex boyfriend',
'exgirlfriend': 'ex girlfriend',
'airhostess': 'air hostess',
'whst': 'what',
'watsapp': 'whatsapp',
'demonitisation': 'demonetization',
'demonitization': 'demonetization',
'demonetisation': 'demonetization',
'quorans': 'quora user',
'quoran': 'quora user',
'pokémon': 'pokemon',
'bacteries': 'batteries',
'yr old': 'years old',
}
codes = ['\x7f', '\u200b', '\xa0', '\ufeff', '\u200e', '\u202a', '\u202c', '\u2060', '\uf0d8', '\ue019', '\uf02d', '\u200f', '\u2061', '\ue01b']
langs1 = r'[\p{Katakana}\p{Hiragana}\p{Han}]'
langs2 = r'[ஆய்தஎழுத்துஆயுதஎழுத்துशुषछछशुषدوउसशुष북한내제តើបងប្អូនមានមធ្យបាយអ្វីខ្លះដើម្បីរកឃើញឯកសារអំពីប្រវត្តិស្ត្រនៃប្រាសាទអង្គរវट्टरौरआदસંઘરાજ્યपीतऊनअहএকটিবাড়িএকটিখামারএরঅধীনেপদেরবাছাইপরীক্ষাএরপ্রশ্নওউত্তরসহকোথায়পেতেপারিص、。Емелядуракلكلمقاممقال수능ί서로가를행복하게기乡국고등학교는몇시간업니《》싱관없어나이रचा키کپڤ」मिलगईकलेजेकोठंडकऋॠऌॡर]'
compiled_langs1 = regex.compile(langs1)
 compiled_langs2 = re.compile(langs2)
BATCH_SIZE = BATCH_SIZE*strategy.num_replicas_in_sync<load_pretrained>
def _clean_math(x, compiled_re):
return compiled_re.sub(' <math> ', x )
MODEL = 'jplu/tf-xlm-roberta-large'
TOKENIZER = AutoTokenizer.from_pretrained(MODEL )<categorify>
def preprocess(x):
x = str(x ).lower()
return x
def _clean_unicode(x):
for u in codes:
if u in x:
x = x.replace(u, '')
 return x
def clean_math(df):
math_puncts = 'θπα÷⁴≠β²¾∫≥⇒¬∠＝∑Φ√½¼'
math_puncts_long = [r'\\frac', r'\[math\]', r'\[/math\]', r'\\lim']
compiled_math = re.compile('(%s)' % '|'.join(math_puncts))
compiled_math_long = re.compile('(%s)' % '|'.join(math_puncts_long))
df['question_text'] = df['question_text'].apply(lambda x: _clean_math(x, compiled_math_long))
df['question_text'] = df['question_text'].apply(lambda x: _clean_math(x, compiled_math))
return df
def clean_abbreviation(df, abbreviations):
compiled_abbreviation = re.compile('(%s)' % '|'.join(abbreviations.keys()))
def replace(match):
return abbreviations[match.group(0)]
df['question_text'] = df["question_text"].apply(
 lambda x: _clean_abreviation(x, compiled_abbreviation, replace)
)
 return df
def _clean_abreviation(x, compiled_re, replace):
return compiled_re.sub(replace, x)
 def clean_spells(df, spells):
compiled_spells = re.compile('(%s)' % '|'.join(spells.keys()))
def replace(match):
return spells[match.group(0)]
df['question_text'] = df["question_text"].apply(
 lambda x: _clean_spells(x, compiled_spells, replace)
)
 return df
def _clean_spells(x, compiled_re, replace):
return compiled_re.sub(replace, x)
 def _clean_language(x, compiled_re):
return compiled_re.sub(' <lang> ', x)
 def _clean_puncts(x, puncts):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
def _clean_space(x, compiled_re):
return compiled_re.sub(" ", x)
def fast_encode(df):
text = df[['premise', 'hypothesis']].values.tolist()
encoded = TOKENIZER.batch_encode_plus(
 text,
pad_to_max_length=True,
max_length=MAX_LEN 
)
 return np.array(encoded["input_ids"] )<categorify>
data["question_text"] = data["question_text"].fillna("" ).apply(preprocess)
 data["question_text"] = data["question_text"].fillna("" ).apply(_clean_unicode)
 data = clean_math(data)
 data = clean_abbreviation(data, abbreviations)
 data = clean_spells(data, spells)
 data['question_text'] = data['question_text'].apply(lambda x: _clean_language(x, compiled_langs1))
data['question_text'] = data['question_text'].apply(lambda x: _clean_language(x, compiled_langs2))
data['question_text'] = data['question_text'].apply(lambda x: _clean_puncts(x, puncts))
compiled_re = re.compile(r"\s+")
 data['question_text'] = data["question_text"].apply(lambda x: _clean_space(x, compiled_re))
test_encoded = fast_encode(test)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_encoded)
.batch(BATCH_SIZE)
 )<feature_engineering>
test_data["question_text"] = test_data["question_text"].fillna("" ).apply(preprocess)
 test_data["question_text"] = test_data["question_text"].fillna("" ).apply(_clean_unicode)
 test_data = clean_math(test_data)
 test_data = clean_abbreviation(test_data, abbreviations)
 test_data = clean_spells(test_data, spells)
 test_data['question_text'] = test_data['question_text'].apply(lambda x: _clean_language(x, compiled_langs1))
test_data['question_text'] = test_data['question_text'].apply(lambda x: _clean_language(x, compiled_langs2))
test_data['question_text'] = test_data['question_text'].apply(lambda x: _clean_puncts(x, puncts))
compiled_re = re.compile(r"\s+")
 test_data['question_text'] = test_data["question_text"].apply(lambda x: _clean_space(x, compiled_re))
del puncts, spells, codes, langs1, langs2, compiled_langs1, compiled_langs2; gc.collect()
def mish(x):
return x*tanh(softplus(x))
get_custom_objects() ["mish"] = Activation(mish )<choose_model_class>
col_names = [ 'q_len', 'q_words', 'u_chars', 'l_chars',
'n_stopwords', 'n_cap_words', 'n_diff_words', 'avg_word_len',
'n_numerical_words', 'nb_simley', 'nb_symbols', 'nb_punct']
data_features = data[col_names]
test_data_features = test_data[col_names]
scaler = StandardScaler().fit(data_features.values)
 train_features = scaler.transform(data_features.values)
 test_features = scaler.transform(test_data_features.values)
 data[col_names] = train_features
test_data[col_names] = test_features
del data_features, test_data_features, train_features, col_names, test_features, scaler; gc.collect()
def create_model(transformer):

 input_ids = Input(shape=(MAX_LEN,), dtype=tf.int32)
 sequence_output = transformer(input_ids)[0]
cls_token = sequence_output[:, 0, :]
cls_token = Dropout(0.3 )(cls_token)
 cls_token = Dense(32, activation='mish' )(cls_token)
 cls_token = Dense(16, activation='mish' )(cls_token)
 out = Dense(3, activation='softmax' )(cls_token)
 optimizer = RectifiedAdam(lr=LR)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 optimizer,
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 return model<choose_model_class>
X_train_question = data['question_text']
X_test_question = test_data['question_text']
tokenizer = Tokenizer()
tokenizer.fit_on_texts(list(X_train_question))
vocab_size = len(tokenizer.word_index)+ 1
print(vocab_size)
 max_features = vocab_size -1
train_encoded_docs = tokenizer.texts_to_sequences(X_train_question)
 test_encoded_docs = tokenizer.texts_to_sequences(X_test_question)
 max_length = 50
train_padded_docs = pad_sequences(train_encoded_docs, maxlen=max_length, padding='post')
 test_padded_docs = pad_sequences(test_encoded_docs, maxlen = max_length, padding='post')
 print(train_padded_docs[0])
 del X_train_question, X_test_question, train_encoded_docs, test_encoded_docs; gc.collect()
kfold = StratifiedKFold(n_splits=NUM_SPLITS, shuffle=True, random_state=SEED )<choose_model_class>
train_padded_docs = pd.DataFrame(train_padded_docs)
 test_padded_docs = pd.DataFrame(test_padded_docs)
 train_data = pd.concat([train_padded_docs, data], axis=1)
 print(train_data.shape)
eas = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=3,
verbose=1, mode='min', baseline=None, restore_best_weights=True )<define_variables>
y = train_data.target
X = train_data.drop(['qid', 'target'], axis=1)
 test_data = test_data.drop(['qid'], axis= 1)
 print(X.shape)
 print(y.shape)
 print(test_data.shape)
oof_preds = np.zeros(( len(train)))
 test_preds = np.zeros(( len(test), 3))<prepare_x_and_y>
del X; gc.collect()
for fold,(train_index, valid_index)in enumerate(kfold.split(train, train['label'])) :
tf.tpu.experimental.initialize_tpu_system(tpu)
 print("*"*60)
 print("*"+" "*26+f"FOLD {fold+1}"+" "*26+"*")
 print("*"*60, end="
")
 X_train = train.iloc[train_index, :].reset_index(drop=True)
 X_valid = train.iloc[valid_index, :].reset_index(drop=True)
 y_train = X_train['label'].values
y_valid = X_valid['label'].values
train_encoded = fast_encode(X_train)
 valid_encoded = fast_encode(X_valid)
 train_dataset = tf.data.Dataset.from_tensor_slices(( train_encoded, y_train))
train_dataset = train_dataset.repeat()
train_dataset = train_dataset.batch(BATCH_SIZE, drop_remainder=True)
 train_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE)
 valid_dataset = tf.data.Dataset.from_tensor_slices(( valid_encoded, y_valid))
valid_dataset = valid_dataset.batch(BATCH_SIZE)
 valid_dataset = valid_dataset.prefetch(tf.data.experimental.AUTOTUNE)
 num_steps = len(X_train)//BATCH_SIZE
with strategy.scope() :
transformer_layer = TFXLMRobertaModel.from_pretrained(MODEL)
 model = create_model(transformer_layer)
 history = model.fit(
 train_dataset,
steps_per_epoch=num_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
callbacks=[eas] 
)
 valid_preds = model.predict(valid_dataset)
 oof_preds[valid_index] = valid_preds.argmax(axis=1)
 preds = model.predict(test_dataset)
 test_preds += preds/NUM_SPLITS<compute_test_metric>
archive = zipfile.ZipFile('.. /input/quora-insincere-questions-classification/embeddings.zip', 'r')
print(f"Accuracy: {accuracy_score(train['label'], oof_preds)}" )<save_to_csv>
def P(word):
"Probability of `word`."
return - WORDS.get(word, 0)
 def correction(word):
"Most probable spelling correction for word."
return max(candidates(word), key=P)
 def candidates(word):
"Generate possible spelling corrections for word."
return(known([word])or known(edits1(word)) or [word])
 def known(words):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in WORDS)
 def edits1(word):
"All edits that are one edit away from `word`."
letters = 'abcdefghijklmnopqrstuvwxyz'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts)
 def edits2(word):
"All edits that are two edits away from `word`."
return(e2 for e1 in edits1(word)for e2 in edits1(e1))
def singlify(word):
return "".join([letter for i,letter in enumerate(word)if i == 0 or letter != word[i-1]] )
submission = pd.read_csv(".. /input/contradictory-my-dear-watson/sample_submission.csv")
 submission['prediction'] = np.argmax(test_preds, axis=1)
 submission.to_csv("submission.csv", index=False )<set_options>
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 embeddings_index = {}
for o in tqdm(archive.open('glove.840B.300d/glove.840B.300d.txt', 'r')) :
o = o.decode("utf-8")
 key , value = get_coefs(*o.split(" "))
embeddings_index.update({key:value})
 words = list(embeddings_index.keys())
 w_rank = {}
for i,word in enumerate(words):
w_rank[word] = i
WORDS = w_rank
plt.style.use('fivethirtyeight')
 warnings.filterwarnings('ignore')
<load_from_csv>
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_1 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
unknown_vector = np.zeros(( 300,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key, i in word_index.items() :
word = key
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_1[i] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_1[i] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_1[i] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_1[i] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_1[i] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_1[i] = embedding_vector
continue
if i> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_1[i] = embedding_vector
continue
embedding_matrix_1[i] = unknown_vector
del embeddings_index
del unknown_vector
gc.collect()
df_train=pd.read_csv(os.path.join(path,"train.csv"))
df_test=pd.read_csv(os.path.join(path,"test.csv"))<set_options>
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_2 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
unknown_vector = np.zeros(( 300,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key, i in word_index.items() :
word = key
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_2[i] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_2[i] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_2[i] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_2[i] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_2[i] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_2[i] = embedding_vector
continue
if i> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_2[i] = embedding_vector
continue
embedding_matrix_2[i] = unknown_vector
del embeddings_index
del unknown_vector
gc.collect()
del data
gc.collect()
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<load_pretrained>
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_3 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
unknown_vector = np.zeros(( 300,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key, i in word_index.items() :
word = key
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_3[i] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_3[i] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_3[i] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_3[i] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_3[i] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_3[i] = embedding_vector
continue
if i> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_3[i] = embedding_vector
continue
embedding_matrix_3[i] = unknown_vector
del embeddings_index
del unknown_vector
gc.collect()
MODEL = 'jplu/tf-xlm-roberta-large'
EPOCHS = 10
MAX_LEN = 96
BATCH_SIZE= 16 * strategy.num_replicas_in_sync
AUTO = tf.data.experimental.AUTOTUNE
tokenizer = AutoTokenizer.from_pretrained(MODEL )<prepare_x_and_y>
embedding_matrix = np.concatenate(( embedding_matrix_1 , embedding_matrix_2, embedding_matrix_3), axis=1)
 del embedding_matrix_1, embedding_matrix_2, embedding_matrix_3;
gc.collect()
def quick_encode(df,maxlen=100):
values = df[['premise','hypothesis']].values.tolist()
tokens=tokenizer.batch_encode_plus(values,max_length=maxlen,pad_to_max_length=True)
 return np.array(tokens['input_ids'])
 x_train = quick_encode(df_train)
 x_test = quick_encode(df_test)
 y_train = df_train.label.values 
<define_variables>
submission_cv = pd.read_csv('.. /input/quora-insincere-questions-classification/sample_submission.csv')
def create_dist_dataset(X, y,val,batch_size= BATCH_SIZE):
dataset = tf.data.Dataset.from_tensor_slices(( X,y)).shuffle(len(X))
if not val:
dataset = dataset.repeat().batch(batch_size ).prefetch(AUTO)
 else:
dataset = dataset.batch(batch_size ).prefetch(AUTO)
 return dataset
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_test)) 
.batch(BATCH_SIZE)
)
<choose_model_class>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 def f1(y_true, y_pred):

 def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
def build_model(transformer,max_len):
input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 sequence_output = transformer(input_ids)[0]
cls_token = sequence_output[:, 0, :]
cls_token = Dropout(0.2 )(cls_token)
 cls_token = Dense(32,activation='relu' )(cls_token)
 out = Dense(3, activation='softmax' )(cls_token)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 return model 
<choose_model_class>
from keras.layers.normalization import BatchNormalization
def build_lrfn(lr_start=0.00001, lr_max=0.00003,
lr_min=0.000001, lr_rampup_epochs=3,
lr_sustain_epochs=0, lr_exp_decay=.6):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn<find_best_params>
input1 = Input(shape=(50,))
x = Embedding(max_features, 300 * 3, weights=[embedding_matrix], input_length= 50, trainable=False, name = 'Embedding' )(input1)
 x = SpatialDropout1D(0.4, name='SpatialDropout' )(x)
 x = Bidirectional(LSTM(256, return_sequences=True), name= 'BidirectionLSTM128' )(x)
 x = Conv1D(64, kernel_size= 1, name='1D_Convolution64' )(x)
 max_pool = GlobalMaxPooling1D(name="GlobalMaxPool" )(x)
 input2 = Input(shape=(12,), name = 'input_stat_featues')
 embed2 = Embedding(22, 40 )(input2)
 conv2 = Conv1D(64, kernel_size= 3, activation='relu', kernel_regularizer= regularizers.l2(0.002), name='Conv1d' )(embed2)
 flatten2 = Flatten(name='Flatten2' )(conv2)
 x = concatenate([max_pool, flatten2], name='Concatenate')
 x = Dense(128, activation="relu", name='1Dense128' )(x)
 x = Dropout(0.1, name='Dropout2' )(x)
 x = BatchNormalization(name='BatchNormalization' )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model1 = Model(inputs=[input1, input2], outputs=x)
 model1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', f1] )
lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1 )<split>
clr = CyclicLR(base_lr=0.001, max_lr=0.002,
step_size=300., mode='exp_range',
gamma=0.99994)
 model1.fit([train_padded_docs, X_train_stat], y, batch_size=512, epochs=4, callbacks = [clr,])
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 val_score=[]
history=[]
for fold,(train_ind,valid_ind)in enumerate(skf.split(x_train,y_train)) :
if fold < 4:
print("fold",fold+1)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 train_data = create_dist_dataset(x_train[train_ind],y_train[train_ind],val=False)
 valid_data = create_dist_dataset(x_train[valid_ind],y_train[valid_ind],val=True)
 Checkpoint=tf.keras.callbacks.ModelCheckpoint(f"roberta_base.h5", monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min')
 with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 n_steps = len(train_ind)//BATCH_SIZE
print("training model {} ".format(fold+1))
train_history = model.fit(
 train_data,
steps_per_epoch=n_steps,
validation_data=valid_data,
epochs=EPOCHS,callbacks=[Checkpoint],verbose=1)
 print("Loading model...")
 model.load_weights(f"roberta_base.h5")
 print("fold {} validation accuracy {}".format(fold+1,np.mean(train_history.history['val_accuracy'])))
 print("fold {} validation loss {}".format(fold+1,np.mean(train_history.history['val_loss'])))
 val_score.append(train_history.history['val_accuracy'])
 history.append(train_history)
 val_score.append(np.mean(train_history.history['val_accuracy']))
print('predict on test.... ')
 preds=model.predict(test_dataset,verbose=1)
 pred_test+=preds/4
print("Mean Validation accuracy : ",np.mean(val_score))<load_from_csv>
train_pred1 = model1.predict([train_padded_docs, X_train_stat], batch_size=512, verbose=1, callbacks = [clr, ])
 test_pred1 = model1.predict([test_padded_docs, test_data1], batch_size=512, verbose=1, callbacks=[clr, ] )
submission = pd.read_csv(os.path.join(path,'sample_submission.csv'))
submission['prediction'] = np.argmax(pred_test,axis=1)
 submission.head()<save_to_csv>
input1 = Input(shape=(50,))
embedding = Embedding(max_features, 300 * 3, weights=[embedding_matrix], input_length= 50, trainable=False, name = 'Embedding' )(input1)
 dropout = SpatialDropout1D(0.4, name='SpatialDropout' )(embedding)
 bidirectionLSTM = Bidirectional(LSTM(128, return_sequences=True), name= 'BidirectionLSTM128' )(dropout)
 bidirectionLSTM2 = Bidirectional(LSTM(128, return_sequences=True), name= 'BidirectionLSTM2' )(bidirectionLSTM)
 max_pool1 = GlobalMaxPooling1D(name="GlobalMaxPool" )(bidirectionLSTM)
 max_pool2 = GlobalMaxPooling1D(name="GlobalMaxPool2" )(bidirectionLSTM2)
 conc = Concatenate()([max_pool1, max_pool2])
 input2 = Input(shape=(12,), name = 'input_stat_featues')
 embed2 = Embedding(12, 50 )(input2)
 conv2 = Conv1D(64, kernel_size= 3, activation='relu', kernel_regularizer= regularizers.l2(0.002), name='Conv1d' )(embed2)
 flatten2 = Flatten(name='Flatten2' )(conv2)
 merge = concatenate([conc, flatten2], name='Concatenate')
 dense64 = Dense(128, activation="relu", name='1Dense128' )(merge)
 dropout2 = Dropout(0.1, name='Dropout2' )(dense64)
 batchnormal = BatchNormalization(name='BatchNormalization' )(dropout2)
 final = Dense(1, activation="sigmoid" )(batchnormal)
 model1 = Model(inputs=[input1, input2], outputs=final)
 model1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', f1])
 model1.fit([train_padded_docs, X_train_stat], y, batch_size=512, epochs= 4, callbacks = [clr,] )
submission.to_csv('submission.csv',index=False )<install_modules>
train_pred2 = model1.predict([train_padded_docs, X_train_stat], batch_size=512, verbose=1, callbacks = [clr, ])
 test_pred2 = model1.predict([test_padded_docs, test_data1], batch_size=512, verbose=1, callbacks=[clr, ] )
!pip install git+https://github.com/ssut/py-googletrans.git<import_modules>
input1 = Input(shape=(50,))
embedding = Embedding(max_features, 300 * 3, weights=[embedding_matrix], input_length= 50, trainable=False, name = 'Embedding' )(input1)
 dropout = SpatialDropout1D(0.3, name='SpatialDropout' )(embedding)
 bidirectionLSTM = Bidirectional(LSTM(256, return_sequences=True), name= 'BidirectionLSTM128' )(dropout)
 x1 = Conv1D(100, activation='relu', kernel_size=1,
padding='same', kernel_initializer= keras.initializers.glorot_uniform(seed=110000))(bidirectionLSTM)
 x2 = Conv1D(80, activation='relu', kernel_size=2,
padding='same', kernel_initializer= keras.initializers.glorot_uniform(seed=120000))(bidirectionLSTM)
 x3 = Conv1D(30, activation='relu', kernel_size=3,
padding='same', kernel_initializer= keras.initializers.glorot_uniform(seed=130000))(bidirectionLSTM)
 x4 = Conv1D(12, activation='relu', kernel_size=5,
padding='same', kernel_initializer= keras.initializers.glorot_uniform(seed=140000))(bidirectionLSTM)
 x1 = GlobalMaxPooling1D()(x1)
 x2 = GlobalMaxPooling1D()(x2)
 x3 = GlobalMaxPooling1D()(x3)
 x4 = GlobalMaxPooling1D()(x4)
 c = concatenate([x1, x2, x3, x4])
 input2 = Input(shape=(12,), name = 'input_stat_featues')
 embed2 = Embedding(12, 40 )(input2)
 conv2 = Conv1D(64, kernel_size= 3, activation='relu', kernel_regularizer= regularizers.l2(0.002), name='Conv1d' )(embed2)
 flatten2 = Flatten(name='Flatten2' )(conv2)
 merge = concatenate([c, flatten2], name='Concatenate')
 x = Dense(128, activation="relu", name='1Dense128' )(merge)
 x = Dropout(0.1, name='Dropout2' )(x)
 x = BatchNormalization(name='BatchNormalization' )(x)
 final = Dense(1, activation="sigmoid" )(x)
 model1 = Model(inputs=[input1, input2], outputs=final)
 model1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', f1] )
import numpy as np
import pandas as pd
from googletrans import Translator
from dask import bag, diagnostics 
<load_from_csv>
model1.fit([train_padded_docs, X_train_stat],
y,
batch_size=512, epochs=4, callbacks = [clr,],
)
train = pd.read_csv('.. /input/contradictory-my-dear-watson/train.csv', index_col=['id'])
 display(train, train.lang_abv.value_counts() )<categorify>
train_pred4 = model1.predict([train_padded_docs, X_train_stat], batch_size= 512, verbose=1, callbacks = [clr, ])
 test_pred4 = model1.predict([test_padded_docs, test_data1], batch_size= 512, verbose=1, callbacks=[clr, ] )
def translate(words, dest):
dest_choices = ['zh-cn',
'ar',
'fr',
'sw',
'ur',
'vi',
'ru',
'hi',
'el',
'th',
'es',
'de',
'tr',
'bg'
]
if not dest:
dest = np.random.choice(dest_choices)
 translator = Translator()
decoded = translator.translate(words, dest=dest ).text
return decoded
def trans_parallel(df, dest):
premise_bag = bag.from_sequence(df.premise.tolist() ).map(translate, dest)
 hypo_bag = bag.from_sequence(df.hypothesis.tolist() ).map(translate, dest)
 with diagnostics.ProgressBar() :
premises = premise_bag.compute()
hypos = hypo_bag.compute()
df[['premise', 'hypothesis']] = list(zip(premises, hypos))
return df
eng = train.loc[train.lang_abv == "en"].copy() \ 
.pipe(trans_parallel, dest=None)
 non_eng = train.loc[train.lang_abv != "en"].copy() \ 
.pipe(trans_parallel, dest='en')
 train = train.append([eng, non_eng])
 train.shape<load_from_csv>
pred_val_y = 0.3 * train_pred1 + 0.3 * train_pred2 + 0.4 * train_pred4
pred_test_y = 0.3 * test_pred1 + 0.3 * test_pred2 + 0.4 * test_pred4
thresholds = []
for thresh in np.arange(0.1, 0.5, 0.01):
thresh = np.round(thresh, 2)
 res = f1_score(y,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )
test = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/test.csv', index_col=['id'])
 submission = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/sample_submission.csv',
index_col=['id'] )<import_modules>
pred_test_y =(pred_test_y > best_thresh ).astype(int )
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
import transformers
from transformers import TFAutoModel, AutoTokenizer<set_options>
submission_cv['prediction'] = pred_test_y
print(submission_cv.head() )
<define_variables><EOS> 
submission_cv.to_csv("submission.csv", index=False )