Kaggle competition: Costa Rican Household Poverty Level Prediction
%%time
x_orig_train = regular_encode(train.orig.values, tokenizer, maxlen=MAX_LEN)
 lang_tag_orig_train = np.array([lang_embed(row['lang'], 'orig')for _, row in train.iterrows() ])
 y_train = train.toxic.values
gap = 128
x_orig_train = np.concatenate(( x_orig_train, x_orig_train[-gap:]))
lang_tag_orig_train = np.concatenate(( lang_tag_orig_train, lang_tag_orig_train[-gap:]))
y_train = np.concatenate(( y_train, y_train[-gap:]))
print(y_train.shape )<load_pretrained>
%matplotlib inline
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.load_weights('/kaggle/input/basemodels/mixmoriggen1.h5')
<train_model>
train= pd.read_csv(".. /input/train.csv")
 test= pd.read_csv(".. /input/test.csv")
 train.head()
history = model.fit([x_orig_train[:63872],lang_tag_orig_train[:63872]], y_train[:63872],
validation_data=([x_valid[:8064],lang_tag_valid[:8064]], y_valid[:8064]),
batch_size=BATCH_SIZE, epochs=1, verbose=1)
<load_pretrained>
train['Target'].value_counts()
model.save_weights("/kaggle/working/mixgn2mp4.h5" )<save_to_csv>
train2=train.drop('Target',axis=1 )
sub['toxic'] = model.predict([x_test, lang_tag_test], verbose=1)
 sub.to_csv('submission.csv', index=False)
 score1 = roc_auc_score(mybest.toxic.round().astype(int), sub.toxic.values)
 score2 = roc_auc_score(sub.toxic.round().astype(int), mybest.toxic.values)
 print('p: %2.4f %2.4f'%(100*score1, 100*score2))<load_from_csv>
data = train2.append(test,sort=True )
submission2 = pd.read_csv('/kaggle/input/009473-v/submission - 2020-06-23T075709.806.csv')
 submission1 = pd.read_csv('/kaggle/input/009488/submission - 2020-06-23T074910.830.csv')
 submission3 = pd.read_csv('.. /input/tpuinference-super-fast-xlmroberta/submission(47 ).csv' )<feature_engineering>
data['dependency'].value_counts()
submission1['toxic'] = submission1['toxic']*0.9 + submission2['toxic']*0.1<save_to_csv>
mapping = {"yes": 1, "no": 0}
data['dependency'] = data['dependency'].replace(mapping ).astype(np.float64)
 data['edjefa'] = data['edjefa'].replace(mapping ).astype(np.float64)
 data['edjefe'] = data['edjefe'].replace(mapping ).astype(np.float64)
 data[['dependency', 'edjefa', 'edjefe']].describe()
submission1.to_csv('submission.csv', index=False )<set_options>
data.loc[data['rez_esc'] == 99.0 , 'rez_esc'] = 5
warnings.simplefilter("ignore" )<categorify>
missing = pd.DataFrame(data.isnull().sum() ).rename(columns = {0: 'total'})
 missing['percent'] = missing['total'] / len(data)
 missing.sort_values('percent', ascending = False ).head(10 )
class TextTransformation:
def __call__(self, text: str, lang: str = None)-> tuple:
raise NotImplementedError('Abstarct')
 class LowerCaseTransformation(TextTransformation):
def __call__(self, text: str, lang: str = None)-> tuple:
return text.lower() , lang
class URLTransformation(TextTransformation):
def __call__(self, text: str, lang: str = None)-> tuple:
for url in self.find_urls(text):
if url in text:
text.replace(url, ' external link ')
 return text.lower() , lang
@staticmethod
def find_urls(string):
urls = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\), ]|(?:%[0-9a-fA-F][0-9a-fA-F])) +', string)
 return urls
class PunctuationTransformation(TextTransformation):
def __call__(self, text: str, lang: str = None)-> tuple:
for p in '?!.,"
if '’' in text:
text = text.replace('’', f' ' ')
 if '’' in text:
text = text.replace('’', f' ' ')
 if '—' in text:
text = text.replace('—', f' - ')
 if '−' in text:
text = text.replace('−', f' - ')
 if '–' in text:
text = text.replace('–', f' - ')
 if '“' in text:
text = text.replace('“', f' " ')
 if '«' in text:
text = text.replace('«', f' " ')
 if '»' in text:
text = text.replace('»', f' " ')
 if '”' in text:
text = text.replace('”', f' " ')
 if '`' in text:
text = text.replace('`', f' ' ')
 text = text.replace(p, f' {p} ')
 return text.strip() , lang
class NumericTransformation(TextTransformation):
def __call__(self, text: str, lang: str = None)-> tuple:
for i in range(10):
text = text.replace(str(i), f' {str(i)} ')
 return text, lang
class WikiTransformation(TextTransformation):
def __call__(self, text: str, lang: str = None)-> tuple:
text = text.replace('wikiproject', ' wiki project ')
 for i in [' vikipedi ', ' wiki ', ' википедии ', " вики ", ' википедия ', ' viki ', ' wikipedien ', ' википедию ']:
text = text.replace(i, ' wikipedia ')
 return text, lang
class MessageTransformation(TextTransformation):
def __call__(self, text: str, lang: str = None)-> tuple:
text = text.replace('wikiproject', ' wiki project ')
 for i in [' msg ', ' msj ', ' mesaj ']:
text = text.replace(i, ' message ')
 return text, lang
class PixelTransformation(TextTransformation):
def __call__(self, text: str, lang: str = None)-> tuple:
for i in [' px ']:
text = text.replace(i, ' pixel ')
 return text, lang
class SaleBotTransformation(TextTransformation):
def __call__(self, text: str, lang: str = None)-> tuple:
text = text.replace('salebot', ' sale bot ')
 return text, lang
class RuTransformation(TextTransformation):
def __call__(self, text: str, lang: str = None)-> tuple:
if lang is not None and lang == 'ru' and 'http' not in text and 'jpg' not in text and 'wikipedia' not in text:
text = text.replace('t', 'т')
 text = text.replace('h', 'н')
 text = text.replace('b', 'в')
 text = text.replace('c', 'c')
 text = text.replace('k', 'к')
 text = text.replace('e', 'е')
 text = text.replace('a', 'а')
 return text, lang
class CombineTransformation(TextTransformation):
def __init__(self, transformations: list, return_lang: bool = False):
self._transformations = transformations
self._return_lang = return_lang
def __call__(self, text: str, lang: str = None)-> tuple:
for transformation in self._transformations:
text, lang = transformation(text, lang)
 if self._return_lang:
return text, lang
return text
def append(self, transformation: TextTransformation):
self._transformations.append(transformation )<categorify>
data['v18q1'] = data['v18q1'].fillna(0)
 data.loc[(data['tipovivi1'] == 1), 'v2a1'] = 0
data['v2a1-missing'] = data['v2a1'].isnull()
data.loc[(( data['age'] > 19)|(data['age'] < 7)) &(data['rez_esc'].isnull()), 'rez_esc'] = 0
data['rez_esc-missing'] = data['rez_esc'].isnull()
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<choose_model_class>
elec = []
for i, row in data.iterrows() :
if row['noelec'] == 1:
elec.append(0)
 elif row['coopele'] == 1:
elec.append(1)
 elif row['public'] == 1:
elec.append(2)
 elif row['planpri'] == 1:
elec.append(3)
 else:
elec.append(np.nan)
 data['elec'] = elec
data['elec-missing'] = data['elec'].isnull()
def build_model(transformer, loss='binary_crossentropy', max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
x = tf.keras.layers.Dropout(0.35 )(cls_token)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=3e-5), loss=loss, metrics=[tf.keras.metrics.AUC() ])
 return model<string_transform>
data = data.drop(columns = ['noelec', 'coopele', 'public', 'planpri'])
nltk.download('punkt')
 stemmer = nltk.stem.porter.PorterStemmer()
remove_punctuation_map = dict(( ord(char), None)for char in string.punctuation)
 def stem_tokens(tokens):
return [stemmer.stem(item)for item in tokens]

 def normalize(text):
return stem_tokens(nltk.word_tokenize(text.lower().translate(remove_punctuation_map)))
 vectorizer = TfidfVectorizer(tokenizer=normalize, stop_words='english')
 def cosine_sim(text1, text2):
tfidf = vectorizer.fit_transform([text1, text2])
 return(( tfidf * tfidf.T ).A)[0,1]<choose_model_class>
data['walls'] = np.argmax(np.array(data[['epared1', 'epared2', 'epared3']]),
axis = 1)
 data = data.drop(columns = ['epared1', 'epared2', 'epared3'] )
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
<load_pretrained>
data['roof'] = np.argmax(np.array(data[['etecho1', 'etecho2', 'etecho3']]),
axis = 1)
 data = data.drop(columns = ['etecho1', 'etecho2', 'etecho3'] )
tokenizer = transformers.BertTokenizer.from_pretrained('bert-base-cased')
 save_path = '/kaggle/working/distilbert_base_uncased/'
if not os.path.exists(save_path):
os.makedirs(save_path)
 tokenizer.save_pretrained(save_path)
 fast_tokenizer = BertWordPieceTokenizer('distilbert_base_uncased/vocab.txt', lowercase=False)
 fast_tokenizer<categorify>
data['floor'] = np.argmax(np.array(data[['eviv1', 'eviv2', 'eviv3']]),
axis = 1)
 data = data.drop(columns = ['eviv1', 'eviv2', 'eviv3'] )
transformer = CombineTransformation(
 [
LowerCaseTransformation() ,
PunctuationTransformation() ,
NumericTransformation() ,
PixelTransformation() ,
MessageTransformation() ,
WikiTransformation() ,
SaleBotTransformation()
]
)<load_from_csv>
data['flush'] = np.argmax(np.array(data[["sanitario1",'sanitario5', 'sanitario2', 'sanitario3',"sanitario6"]]),
axis = 1)
 data = data.drop(columns = ["sanitario1",'sanitario5', 'sanitario2', 'sanitario3',"sanitario6"] )
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train1['comment_text'] = train1.apply(lambda x: transformer(x.comment_text), axis=1)
 valid1 = pd.read_csv('/kaggle/input/val-en-df/validation_en.csv')
 valid1['comment_text'] = valid1.apply(lambda x: transformer(x.comment_text_en), axis=1)
 valid2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_valid_translated.csv')
 valid2['comment_text'] = valid2.apply(lambda x: transformer(x.translated), axis=1)
 test1 = pd.read_csv('/kaggle/input/test-en-df/test_en.csv')
 test1['comment_text'] = test1.apply(lambda x: transformer(x.content_en), axis=1)
 test2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv')
 test2['comment_text'] = test2.apply(lambda x: transformer(x.translated), axis=1)
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<categorify>
data = data[[x for x in data if not x.startswith('SQB')]]
data = data.drop(columns = ['agesq'] )
x_train = fast_encode(train1.comment_text.astype(str), fast_tokenizer, maxlen=512)
 x_valid1 = fast_encode(valid1.comment_text_en.astype(str), fast_tokenizer, maxlen=512)
 x_valid2 = fast_encode(valid2.translated.astype(str), fast_tokenizer, maxlen=512)
 x_valid = np.concatenate([x_valid1, x_valid2], axis=0)
 x_test1 = fast_encode(test1.content_en.astype(str), fast_tokenizer, maxlen=512)
 x_test2 = fast_encode(test2.translated.astype(str), fast_tokenizer, maxlen=512)
 y_train = train1.toxic.values
y_valid1 = valid1.toxic.values
y_valid2 = valid2.toxic.values
y_valid = np.concatenate([y_valid1, y_valid2], axis=0 )<categorify>
data['waterprovision'] = np.argmax(np.array(data[['abastaguano', 'abastaguafuera', 'abastaguadentro']]),
axis = 1)
 data = data.drop(columns = ['abastaguano', 'abastaguafuera', 'abastaguadentro'] )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(64)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(64)
.cache() 
.prefetch(AUTO)
)
 test_dataset = [(
 tf.data.Dataset 
.from_tensor_slices(x_test1)
.batch(64)
), 
(
 tf.data.Dataset 
.from_tensor_slices(x_test2)
.batch(64)
)]<compute_train_metric>
data['inst'] = np.argmax(np.array(data[[c for c in data if c.startswith('instl')]]), axis = 1)
 data = data.drop(columns = [c for c in data if c.startswith('instlevel')])
def focal_loss(gamma=2., alpha=.2):
def focal_loss_fixed(y_true, y_pred):
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
return -K.mean(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) - K.mean(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return focal_loss_fixed<load_pretrained>
data['waterprovision'] = np.argmax(np.array(data[['energcocinar1','energcocinar4', 'energcocinar3', 'energcocinar2']]),
axis = 1)
 data = data.drop(columns = ['energcocinar1','energcocinar4', 'energcocinar3', 'energcocinar2'] )
%%time
with strategy.scope() :
transformer_layer = transformers.TFBertModel.from_pretrained('bert-base-cased')
 model = build_model(transformer_layer, loss=focal_loss(gamma=1.5), max_len=512)
 model.summary()<choose_model_class>
data.loc[pd.isnull(data['meaneduc']), 'meaneduc'] = data.loc[pd.isnull(data['meaneduc']), 'escolari']
class RocAucCallback(Callback):
def __init__(self, test_data, score_thr):
self.test_data = test_data
self.score_thr = score_thr
self.test_pred = []
def on_epoch_end(self, epoch, logs=None):
if logs['val_auc'] > self.score_thr:
print('
Run TTA...')
 for td in self.test_data:
self.test_pred.append(self.model.predict(td))<choose_model_class>
train2=data.iloc[0:9557,:]
test2=data.iloc[9557:33413,:]
def build_lrfn(lr_start=0.000001, lr_max=0.000004,
lr_min=0.0000001, lr_rampup_epochs=7,
lr_sustain_epochs=0, lr_exp_decay=.87):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn<train_on_grid>
test2.drop(['Id','idhogar'],axis=1,inplace=True )
roc_auc = RocAucCallback(test_dataset, 0.935)
 lrfn = build_lrfn()
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 train_history = model.fit(
 train_dataset,
steps_per_epoch=150,
validation_data=valid_dataset,
callbacks=[lr_schedule, roc_auc],
epochs=35
)<import_modules>
X=train2.drop(['Id','idhogar'],axis=1 )
from sklearn.model_selection import train_test_split<import_modules>
y=train['Target']
from sklearn.model_selection import train_test_split<split>
import xgboost as xgb
x_valid_train1, x_valid_valid1, y_valid_train1, y_valid_valid1 = train_test_split(x_valid1, y_valid1, test_size=0.15, shuffle=True, random_state=123, stratify=y_valid1)
 x_valid_train2, x_valid_valid2, y_valid_train2, y_valid_valid2 = train_test_split(x_valid2, y_valid2, test_size=0.15, shuffle=True, random_state=123, stratify=y_valid2)
 x_valid_train = np.concatenate([x_valid_train1, x_valid_train2], axis=0)
 y_valid_train = np.concatenate([y_valid_train1, y_valid_train2], axis=0)
 x_valid_valid = np.concatenate([x_valid_valid1, x_valid_valid2], axis=0)
 y_valid_valid = np.concatenate([y_valid_valid1, y_valid_valid2], axis=0)
 valid_train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid_train, y_valid_train)) 
.shuffle(2048)
.batch(64)
.cache() 
.prefetch(AUTO)
)
 valid_valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid_valid, y_valid_valid)) 
.batch(64)
.cache() 
.prefetch(AUTO)
 )<train_model>
xg=xgb.XGBClassifier(n_estimators=200 )
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 train_history = model.fit(
 valid_train_dataset,
steps_per_epoch=75,
validation_data=valid_valid_dataset,
callbacks=[lr_schedule, roc_auc],
epochs=25
)<save_to_csv>
xg.fit(X,y )
sub['toxic'] = np.mean(roc_auc.test_pred, axis=0)
 sub.to_csv('submission.csv', index=False )<import_modules>
preds = xg.predict(test2 )
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from tqdm.notebook import tqdm
from tokenizers import BertWordPieceTokenizer, SentencePieceBPETokenizer<set_options>
def macro_f1_score(
 labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average = 'macro')
 return 'macro_f1', metric_value, True
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
import lightgbm as lgb
import sklearn.model_selection as model_selection
from sklearn.metrics import f1_score, make_scorer
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')
 EPOCHS = 3
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 128<load_pretrained>
lgmodel = lgb.LGBMClassifier(metric = "",num_class = 4 )
tokenizer = transformers.AutoTokenizer.from_pretrained("bert-base-cased")
 tokenizer.save_pretrained('.')
<load_from_csv>
hyp_OPTaaS = { 'boosting_type': 'dart',
'colsample_bytree': 0.9843467236959204,
'learning_rate': 0.11598629586769524,
'min_child_samples': 44,
'num_leaves': 49,
'reg_alpha': 0.35397370408131534,
'reg_lambda': 0.5904910774606467,
'subsample': 0.6299872254632797,
'subsample_for_bin': 60611}
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
<count_values>
model = lgb.LGBMClassifier(**hyp_OPTaaS, class_weight = 'balanced',max_depth=-1,objective = 'multiclass', n_jobs = -1, n_estimators = 100 )
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=150000, random_state=0)
 ])
 print(train.toxic.value_counts())
 print(valid.toxic.value_counts() )<load_from_csv>
model.fit(X, y )
olid = pd.read_csv('/kaggle/input/olid2019/olid-training-v1.0.tsv', sep='\t')
 olid = olid.rename(columns={"id": "id", "tweet": "comment_text", 'subtask_a': 'toxic'})
 olid.toxic =(olid.toxic == 'OFF' ).astype(int)
 olid.head()<load_from_csv>
pred=model.predict(test2 )
<count_values><EOS> 
my_submission = pd.DataFrame({'Id': test.Id, 'Target': pred})
 my_submission.to_csv('submission.csv', index=False )