Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<define_variables>
warnings.filterwarnings("ignore")
 tqdm.pandas()
CountVectorizer,\
HashingVectorizer
stopword=set(STOPWORDS)
 lem = WordNetLemmatizer()
tokenizer=TweetTokenizer()
np.random.seed(0)
extra_drop_features = ['geo_manual_elec_LE_2',
'agg18_estadocivil5_COUNT',
'agg18_estadocivil7_COUNT',
'geo_pared_LE_5',
'agg18_estadocivil6_COUNT',
'geo_pared_LE_4',
'agg18_parentesco10_COUNT',
'num_over_18',
'agg18_estadocivil1_MEAN',
'agg18_estadocivil4_COUNT',
'r4t3_to_tamhog',
'agg18_estadocivil3_COUNT',
'geo_pared_LE_3',
'geo_pared_LE_6',
'geo_pared_LE_0',
'geo_pared_LE_7',
'agg18_parentesco7_COUNT',
'agg18_parentesco10_MEAN',
'agg18_parentesco8_MEAN',
'parentesco1',
'rez_esc',
'dependency_yes',
'hacdor',
'hacapo',
'fe_people_weird_stat',
'geo_hogar_mayor',
'geo_eviv_LE_0',
'geo_epared_LE_0',
'geo_epared_LE_2',
'geo_elimbasu_LE_4',
'geo_energcocinar_LE_0',
'agg18_parentesco9_COUNT',
'agg18_parentesco8_COUNT',
'agg18_parentesco11_COUNT',
'parentesco_LE',
'agg18_parentesco6_MEAN',
'agg18_parentesco6_COUNT',
'agg18_parentesco5_COUNT',
'agg18_parentesco4_MEAN',
'agg18_parentesco4_COUNT',
'geo_energcocinar_LE_1',
'agg18_parentesco3_COUNT',
'geo_energcocinar_LE_2',
'agg18_parentesco2_COUNT',
'agg18_parentesco1_COUNT',
'agg18_parentesco12_MEAN',
'agg18_parentesco12_COUNT',
'dependency_no'] + ['geo_hogar_nin', 'geo_manual_elec_LE_3']<drop_column>
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print(strategy.num_replicas_in_sync)
 BATCH_SIZE = 16 * strategy.num_replicas_in_sync
lgb_drop_cols = extra_drop_features + ["idhogar"]
try:
X_train.drop(lgb_drop_cols, axis=1, inplace=True)
 X_test.drop(lgb_drop_cols, axis=1, inplace=True)
 except:
print("Error dropping columns" )<compute_train_metric>
MODEL = 'jplu/tf-xlm-roberta-base'
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
 text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
tokenized_text = tokenizer.tokenize(text )
opt_parameters = {'colsample_bytree': 0.93, 'min_child_samples': 56, 'num_leaves': 19, 'subsample': 0.84}
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 fit_params={"early_stopping_rounds":300,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(X_train,y_train),(X_test,y_test)],
'eval_names': ['train', 'valid'],
'verbose': False,
'categorical_feature': 'auto'}
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params['verbose'] = 200
fit_params['callbacks'] = [lgb.reset_parameter(learning_rate=learning_rate_power_0997)]<train_model>
masked_index = 8
tokenized_text[masked_index] = '[MASK]'
assert tokenized_text == ['[CLS]', 'who', 'was', 'jim', 'henson', '?', '[SEP]', 'jim', '[MASK]', 'was', 'a', 'puppet', '
indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
 segments_ids = [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1]
tokens_tensor = torch.tensor([indexed_tokens])
 segments_tensors = torch.tensor([segments_ids] )
def _parallel_fit_estimator(estimator, X, y, sample_weight=None, **fit_params):
X_train, y_train, X_test, y_test = split_data(X, y, households=train_households, seed=None)
 fit_params["eval_set"] = [(X_train,y_train),(X_test,y_test)]

 if sample_weight is not None:
estimator.fit(X_train, y_train, sample_weight=sample_weight, **fit_params)
 else:
estimator.fit(X_train, y_train, **fit_params)
 return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 if sample_weight is not None:
for name, step in self.estimators:
if not has_fit_parameter(step, 'sample_weight'):
raise ValueError('Underlying estimator '%s' does not'
' support sample weights.' % name)
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y,
sample_weight=sample_weight, **fit_params)
 for clf in clfs if clf is not None)
 return self<train_model>
model = BertForMaskedLM.from_pretrained('bert-base-uncased')
 model.eval()
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print(device)
 tokens_tensor = tokens_tensor.to(device)
 segments_tensors = segments_tensors.to(device)
 model.to(device)
 with torch.no_grad() :
outputs = model(tokens_tensor, token_type_ids=segments_tensors)
 predictions = outputs[0]
predicted_index = torch.argmax(predictions[0, masked_index] ).item()
predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0]
assert predicted_token == 'henson'
print('Predicted token is:',predicted_token )
clfs = []
for i in range(10):
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=314+i, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced')
 clf.set_params(**opt_parameters)
 clfs.append(( 'lgbm{}'.format(i), clf))
vc = VotingClassifierLGBM(clfs, voting='soft')
 del clfs
_ = vc.fit(X_train, y_train, **fit_params)
 clf_final = vc.estimators_[0]<compute_train_metric>
import pandas as pd
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test), average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test), average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))<compute_train_metric>
import pandas as pd
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test), average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test), average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))<compute_test_metric>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 train3 = pd.read_csv('.. /input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv')
 train4 = pd.read_csv('.. /input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv')
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 toxic = len(train2[['comment_text', 'toxic']].query('toxic==1'))
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train3[['comment_text', 'toxic']].query('toxic==0'),
train3[['comment_text', 'toxic']].query('toxic==1'),
train4[['comment_text', 'toxic']].query('toxic==0'),
train4[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=(toxic+(toxic//3)) , random_state=101)
 ])
 test_data = test
train_data = train
maxlen = 96
ranked_features = feature_importance(clf_final, X_train )<create_dataframe>
df_valid_en = pd.read_csv('.. /input/val-en-df/validation_en.csv')
 df_valid_en = df_valid_en.drop(columns=['id', 'comment_text','lang'])
 df_valid_en = df_valid_en.rename(columns={"comment_text_en": "comment_text"})
 columns_titles = ["comment_text","toxic"]
df_valid_en=df_valid_en.reindex(columns=columns_titles)

print(df_valid_en.head(3))
train_data = pd.concat([train_data , df_valid_en], axis=0 ).reset_index(drop=True)
 train_data = train_data.sample(frac=1 ).reset_index(drop=True)
 print(len(train2[['comment_text', 'toxic']].query('toxic==1')))
 print(len(train_data))
train_data.head()
y_subm = pd.DataFrame()
y_subm['Id'] = test_ids<predict_on_test>
val = valid
train = train_data
def clean(text):
text = text.fillna("fillna" ).str.lower()
text = text.map(lambda x: re.sub('\
',' ',str(x)))
 text = text.map(lambda x: re.sub("\[\[User.*",'',str(x)))
 text = text.map(lambda x: re.sub("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}",'',str(x)))
 text = text.map(lambda x: re.sub("\(http://.*?\s\(http://.*\)",'',str(x)))
 return text
val["comment_text"] = clean(val["comment_text"])
 test_data["content"] = clean(test_data["content"])
 train["comment_text"] = clean(train["comment_text"] )
y_subm['Target'] = clf_final.predict(test[X_train.columns])+ 1
vc.voting = 'soft'
y_subm_soft = y_subm.copy(deep=True)
 y_subm_soft['Target'] = vc.predict(test[X_train.columns])+ 1
vc.voting = 'hard'
y_subm_hard = y_subm.copy(deep=True)
 y_subm_hard['Target'] = vc.predict(test[X_train.columns])+ 1<save_to_csv>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '\xa0', '\t',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '\u3000', '\u202f',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '«',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"couldnt" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"doesnt" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"havent" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"shouldnt" : "should not",
"that's" : "that is",
"thats" : "that is",
"there's" : "there is",
"theres" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"theyre": "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def clean_text(x):
x = str(x ).replace("
","")
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
now = datetime.now()
sub_file = 'submission_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_soft = 'submission_soft_LGB_{:.4f}_{}.csv'.format(global_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_hard = 'submission_hard_LGB_{:.4f}_{}.csv'.format(global_score_hard, str(now.strftime('%Y-%m-%d-%H-%M')))
 y_subm_soft.to_csv(sub_file_soft, index=False)
 y_subm_hard.to_csv(sub_file_hard, index=False )<set_options>
tokenizer = TreebankWordTokenizer()
def handle_contractions(x):
x = tokenizer.tokenize(x)
 return x
def fix_quote(x):
x = [x_[1:] if x_.startswith("'")else x_ for x_ in x]
x = ' '.join(x)
 return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
def replace_typical_misspell(text):
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def clean_data(df, columns: list):
for col in columns:
df[col] = df[col].apply(lambda x: clean_text(x.lower()))
df[col] = df[col].apply(lambda x: replace_typical_misspell(x))
df[col] = df[col].apply(lambda x: handle_contractions(x))
df[col] = df[col].apply(lambda x: fix_quote(x))
return df
%matplotlib inline
%reload_ext autoreload
%autoreload 2
np.set_printoptions(threshold=50, edgeitems=20)
 pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
<load_from_csv>
%%time
input_columns = [
'comment_text'
]

 train = clean_data(train, input_columns)
 val = clean_data(val, input_columns)
PATH = '.. /input/costa-rican-household-poverty-prediction/'
train_df = pd.read_csv(PATH+'train.csv')
 test_df = pd.read_csv(PATH+'test.csv' )<define_variables>
%%time
input_columns = [
'content'
]
test_data = clean_data(test_data, input_columns)
 del tokenizer
cat_vars = ['hacdor',
'hacapo',
'v14a',
'refrig',
'v18q',
'paredblolad',
'paredzocalo',
'paredpreb',
'pareddes',
'paredmad',
'paredzinc',
'paredfibras',
'paredother',
'pisomoscer',
'pisocemento',
'pisoother',
'pisonatur',
'pisonotiene',
'pisomadera',
'techozinc',
'techoentrepiso',
'techocane',
'techootro',
'cielorazo',
'abastaguadentro',
'abastaguafuera',
'abastaguano',
'public',
'planpri',
'noelec',
'coopele',
'sanitario1',
'sanitario2',
'sanitario3',
'sanitario5',
'sanitario6',
'energcocinar1',
'energcocinar2',
'energcocinar3',
'energcocinar4',
'elimbasu1',
'elimbasu2',
'elimbasu3',
'elimbasu4',
'elimbasu5',
'elimbasu6',
'epared1',
'epared2',
'epared3',
'etecho1',
'etecho2',
'etecho3',
'eviv1',
'eviv2',
'eviv3',
'dis',
'male',
'female',
'estadocivil1',
'estadocivil2',
'estadocivil3',
'estadocivil4',
'estadocivil5',
'estadocivil6',
'estadocivil7',
'parentesco1',
'parentesco2',
'parentesco3',
'parentesco4',
'parentesco5',
'parentesco6',
'parentesco7',
'parentesco8',
'parentesco9',
'parentesco10',
'parentesco11',
'parentesco12',
'dependency',
'edjefe',
'edjefa',
'instlevel1',
'instlevel2',
'instlevel3',
'instlevel4',
'instlevel5',
'instlevel6',
'instlevel7',
'instlevel8',
'instlevel9',
'tipovivi1',
'tipovivi2',
'tipovivi3',
'tipovivi4',
'tipovivi5',
'computer',
'television',
'mobilephone',
'lugar1',
'lugar2',
'lugar3',
'lugar4',
'lugar5',
'lugar6',
'area1',
'area2',
'idhogar']
contin_vars = ['v2a1',
'rooms',
'v18q1',
'r4h1',
'r4h2',
'r4h3',
'r4m1',
'r4m2',
'r4m3',
'r4t1',
'r4t2',
'r4t3',
'tamhog',
'escolari',
'rez_esc',
'hhsize',
'hogar_nin',
'hogar_adul',
'hogar_mayor',
'hogar_total',
'meaneduc',
'bedrooms',
'overcrowding',
'qmobilephone',
'age',
'SQBescolari',
'SQBage',
'SQBhogar_total',
'SQBedjefe',
'SQBhogar_nin',
'SQBovercrowding',
'SQBdependency',
'SQBmeaned',
'agesq']
objective= ['Target']
ids = ['Id']<data_type_conversions>
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))
for v in cat_vars: train_df[v] = train_df[v].astype('category' ).cat.as_ordered()
apply_cats(test_df, train_df )<data_type_conversions>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )
for v in contin_vars:
train_df[v] = train_df[v].fillna(0 ).astype('float32')
 test_df[v] = test_df[v].fillna(0 ).astype('float32' )<create_dataframe>
MODEL = 'jplu/tf-xlm-roberta-large'
tokenizer = AutoTokenizer.from_pretrained(MODEL)
 save_path = '/kaggle/working/xlmr_large/'
if not os.path.exists(save_path):
os.makedirs(save_path)
 tokenizer.save_pretrained(save_path)
for dataframe in(train_df, test_df):
dataframe['Rent_per_individual'] = dataframe['v2a1']/dataframe['r4t3']
dataframe['Rent_per_child'] = dataframe['v2a1']/dataframe['r4t1']
dataframe['Rent_per_over_65'] = dataframe['v2a1']/dataframe['r4t3']
dataframe['Rent_per_room'] = dataframe['v2a1']/dataframe['rooms']
dataframe['Rent_per_bedrooms'] = dataframe['v2a1']/dataframe['bedrooms']
dataframe['Proportion_under_12'] = dataframe['r4t1']/dataframe['r4t3']
dataframe['Proportion_under_12_male'] = dataframe['r4h1']/dataframe['r4t3']
dataframe['Proportion_under_12_female'] = dataframe['r4m1']/dataframe['r4t3']
dataframe['Proportion_male'] = dataframe['r4h3']/dataframe['r4t3']
dataframe['Proportion_female'] = dataframe['r4m3']/dataframe['r4t3']
dataframe['Rooms_per_individual'] = dataframe['rooms']/dataframe['r4t3']
dataframe['Rooms_per_child'] = dataframe['rooms']/dataframe['r4t1']
dataframe['Tablets_per_individual'] = dataframe['v18q1']/dataframe['r4t3']
dataframe['Tablets_per_child'] = dataframe['v18q1']/dataframe['r4t1']
dataframe['Years_schooling_per_individual'] = dataframe['escolari']/dataframe['r4t3']
dataframe['Years_schooling_per_adult'] = dataframe['escolari']/(dataframe['r4t3']-dataframe['r4t1'])
 dataframe['Years_schooling_per_child'] = dataframe['escolari']/dataframe['r4t1']
dataframe['Proportion_under_19'] = dataframe['hogar_nin']/dataframe['r4t3']
dataframe['Proportion_over_19'] = dataframe['hogar_adul']/dataframe['r4t3']
dataframe['Proportion_under_65'] =(dataframe['hogar_total']-dataframe['hogar_mayor'])/dataframe['r4t3']
dataframe['Proportion_over_65'] = dataframe['hogar_mayor']/dataframe['r4t3']
dataframe['Bedrooms_per_individual'] = dataframe['bedrooms']/dataframe['r4t3']
dataframe['Bedrooms_per_child'] = dataframe['bedrooms']/dataframe['r4t1']
dataframe['Bedrooms_per_over_65'] = dataframe['bedrooms']/dataframe['r4t3']
dataframe['Extreme_conditions_flag'] =(dataframe['abastaguano'] & dataframe['noelec'] & dataframe['sanitario1'] & dataframe['energcocinar1'])
 dataframe['bedrooms_to_rooms'] = dataframe['bedrooms']/dataframe['rooms']
dataframe['tamhog_to_rooms'] = dataframe['tamhog']/dataframe['rooms']
dataframe['tamhog_to_bedrooms'] = dataframe['tamhog']/dataframe['bedrooms']
dataframe['r4t3_to_tamhog'] = dataframe['r4t3']/dataframe['tamhog']
dataframe['hhsize_to_rooms'] = dataframe['hhsize']/dataframe['rooms']
dataframe['hhsize_to_bedrooms'] = dataframe['hhsize']/dataframe['bedrooms']
dataframe['rent_to_hhsize'] = dataframe['v2a1']/dataframe['hhsize']
dataframe['qmobilephone_to_r4t3'] = dataframe['qmobilephone']/dataframe['r4t3']
dataframe['qmobilephone_to_v18q1'] = dataframe['qmobilephone']/dataframe['v18q1']<define_variables>
%%time
x_train = regular_encode(train.comment_text.astype(str),
tokenizer, maxlen=maxlen)
 x_valid = regular_encode(val.comment_text.astype(str ).values,
tokenizer, maxlen=maxlen)
 x_test = regular_encode(test_data.content.astype(str ).values,
tokenizer, maxlen=maxlen)
 y_valid = val.toxic.values
y_train = train.toxic.values
new_features = ['Rent_per_individual',
'Rent_per_child',
'Rent_per_over_65',
'Rent_per_room',
'Rent_per_bedrooms',
'Proportion_under_12',
'Proportion_under_12_male',
'Proportion_under_12_female',
'Proportion_male',
'Proportion_female',
'Rooms_per_individual',
'Rooms_per_child',
'Tablets_per_individual',
'Tablets_per_child',
'Years_schooling_per_individual',
'Years_schooling_per_adult',
'Years_schooling_per_child',
'Proportion_under_19',
'Proportion_over_19',
'Proportion_under_65',
'Proportion_over_65',
'Bedrooms_per_individual',
'Bedrooms_per_child',
'Bedrooms_per_over_65',
'Extreme_conditions_flag',
'bedrooms_to_rooms',
'tamhog_to_rooms',
'tamhog_to_bedrooms',
'r4t3_to_tamhog',
'hhsize_to_rooms',
'hhsize_to_bedrooms',
'rent_to_hhsize',
'qmobilephone_to_r4t3',
'qmobilephone_to_v18q1']
for v in new_features:
train_df[v] = train_df[v].fillna(0 ).astype('float32')
 test_df[v] = test_df[v].fillna(0 ).astype('float32' )<prepare_x_and_y>
%%time
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
train_df = train_df.set_index('Id')
 train_df = train_df[cat_vars+contin_vars+objective]
train_X, train_y, nas, mapper = proc_df(train_df, 'Target', do_scale=True )<categorify>
def focal_loss(gamma=2., alpha=.2):
def focal_loss_fixed(y_true, y_pred):
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
return -K.mean(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) - K.mean(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return focal_loss_fixed
test_df = test_df.set_index('Id')
 test_df['Target'] = 0
test_df = test_df[cat_vars+contin_vars+['Target']]
test_X, _, nas, mapper = proc_df(test_df, 'Target', do_scale=True,
mapper=mapper, na_dict=nas )<init_hyperparams>
def build_model(transformer, loss='binary_crossentropy', max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
x = tf.keras.layers.Dropout(0.2 )(cls_token)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=3e-5), loss=loss, metrics=[tf.keras.metrics.AUC() ])
 return model
opt_params = {'bagging_fraction': 0.8116167224336399,
'colsample_bytree': 0.8368209952651108,
'feature_fraction': 0.5789267873576293,
'lambda_l1': 8.324426408004218,
'lambda_l2': 1.8340450985343382,
'max_depth': 12,
'min_child_samples': 71,
'min_child_weight': 15,
'min_split_gain': 0.08695705870978104,
'num_leaves': 49,
'subsample': 0.8822634555704315}<load_from_csv>
%%time
with strategy.scope() :
transformer_layer = transformers.TFXLMRobertaModel.from_pretrained(MODEL)
 model = build_model(transformer_layer,loss='binary_crossentropy', max_len=maxlen)
 model.summary()
<set_options>
def callback() :
cb = []
reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss',
factor=0.3, patience=2,
verbose=1, mode='auto',
epsilon=0.0001, cooldown=1, min_lr=0.000001)
 cb.append(reduceLROnPlat)
 log = CSVLogger('log.csv')
 cb.append(log)
 RocAuc = RocAucEvaluation(validation_data=(x_valid, y_valid), interval=1)
 cb.append(RocAuc)
 return cb
%matplotlib inline 
<load_from_csv>
def build_lrfn(lr_start=0.000001, lr_max=0.000002,
lr_min=0.0000001, lr_rampup_epochs=7,
lr_sustain_epochs=0, lr_exp_decay=.87):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
test = pd.read_csv(".. /input/test.csv")
 poverty_train = pd.read_csv(".. /input/train.csv")
 display(poverty_train.head() , poverty_train.shape )<prepare_output>
model_path = 'jigsawMultilingual.hdf5'
model_path1 = '/kaggle/working/jigsawMultilingual.hdf5'
print("Top Columns having missing values in the train set.")
 missmap = poverty_train.isnull().sum().to_frame().sort_values(0, ascending = False)/ len(poverty_train)
 missmap.head()<categorify>
checkpoint = ModelCheckpoint(model_path, monitor='val_accuracy', mode='max', save_best_only=True)
 es = EarlyStopping(monitor='val_accuracy', mode='max', patience=2,
restore_best_weights=True, verbose=1)
 lr_callback = LearningRateScheduler(lrfn, verbose=1)
 callback_list = [checkpoint, lr_callback]
print("Top Columns having missing values in the test set.")
 missmap = test.isnull().sum().to_frame().sort_values(0, ascending = False)/ len(test)
 missmap.head()<filter>
%%time
N_STEPS = x_train.shape[0] // BATCH_SIZE
EPOCHS = 2
train_history = model.fit(
 train_dataset,
steps_per_epoch=N_STEPS,
validation_data=valid_dataset,
callbacks=callback_list,
epochs=EPOCHS
)
poverty_train[poverty_train.meaneduc.isnull() ]<compute_test_metric>
if os.path.exists(model_path1):
model.load_weights(model_path1 )
value1 = poverty_train["meaneduc"].mean()
value2 = value1*value1
display(value1, value2)
<categorify>
%%time
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
callbacks=callback_list,
epochs= EPOCHS
)
poverty_train["meaneduc"].fillna(value1, inplace = True)
 poverty_train["SQBmeaned"].fillna(value2, inplace = True)
 print("Top Columns having missing values")
 missmap = poverty_train.isnull().sum().to_frame().sort_values(0, ascending = False)/ len(poverty_train)
 missmap.head()<count_missing_values>
model.save_weights('model.h5' )
display(poverty_train.loc[(poverty_train["tipovivi1" ] == 1, "v2a1")].isna().sum() ,
len(poverty_train.loc[(poverty_train["tipovivi1" ] == 1, "v2a1")]))<feature_engineering>
if os.path.exists(model_path1):
model.load_weights(model_path1 )
poverty_train.loc[(poverty_train["tipovivi1" ] == 1, "v2a1")] = 0
print("Top Columns having missing values")
 missmap = poverty_train.isnull().sum().to_frame().sort_values(0, ascending = False)/ len(poverty_train)
 missmap.head()<count_missing_values>
log_dir = "/kaggle/working/log.csv"
if os.path.exists(log_dir):
os.remove(log_dir )
<count_missing_values><EOS> 
sub = pd.read_csv('.. /input/jigsaw-multilingual-toxic-comment-classification/' + 'sample_submission.csv')
 sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )