Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<define_variables>
len(set(weird))<filter>
interativeEnvironment=False
localEnvironment=False
IN_COLAB=False
try:
except ImportError as e:
try:
IN_COLAB = True
except:
IN_COLAB = False
localEnvironment=True
os.environ['XLA_USE_BF16'] = "1"
localEnvironment
pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', 300)
 pd.set_option('display.width', None)
 %matplotlib inline
data[data['idhogar']==weird[2]][['idhogar','parentesco1', 'Target']]<feature_engineering>
def is_interactive() :
return 'runtime' in get_ipython().config.IPKernelApp.connection_file
print('Interactive?', is_interactive() )
for i in set(weird):
hhold=data[data['idhogar']==i][['idhogar', 'parentesco1', 'Target']]
target=hhold[hhold['parentesco1']==1]['Target'].tolist() [0]
for row in hhold.iterrows() :
idx=row[0]
if row[1]['parentesco1']!=1:
data.at[idx, 'Target']=target 
<filter>
data[data['idhogar']==weird[1]][['idhogar','parentesco1', 'Target']]<categorify>
tf_env="tensorflow"
def data_cleaning(data):
data['dependency']=np.sqrt(data['SQBdependency'])
 data['rez_esc']=data['rez_esc'].fillna(0)
 data['v18q1']=data['v18q1'].fillna(0)
 data['v2a1']=data['v2a1'].fillna(0)
 conditions = [ 
(data['edjefe']=='no')&(data['edjefa']=='no'), 
(data['edjefe']=='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']=='yes'), 
(data['edjefe']!='no')&(data['edjefe']!='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']!='no')
 ]
choices = [0, 1, 1, data['edjefe'], data['edjefa']]
data['edjefx']=np.select(conditions, choices)
 data['edjefx']=data['edjefx'].astype(int)
 data.drop(['edjefe', 'edjefa'], axis=1, inplace=True)
 meaneduc_nan=data[data['meaneduc'].isnull() ][['Id','idhogar','escolari']]
me=meaneduc_nan.groupby('idhogar')['escolari'].mean().reset_index()
for row in meaneduc_nan.iterrows() :
idx=row[0]
idhogar=row[1]['idhogar']
m=me[me['idhogar']==idhogar]['escolari'].tolist() [0]
data.at[idx, 'meaneduc']=m
data.at[idx, 'SQBmeaned']=m*m
return data<set_options>
fit_verbocity=1
model=tf_env
nrows=None
interativeEnvironment=is_interactive()
interativeEnvironment=False
if localEnvironment:
warnings.warn("Nrows limited")
 pass
elif interativeEnvironment:
pass
else:
fit_verbocity=2
pooling_mode_cls_token="CLS_TOKEN"
pooling_mode_fc="FC"
pooling_mode=pooling_mode_fc
fc_dims=[]
trainable_transformer=False
use_dann=False
dann_lambda=-1
use_parcor=False
parcor_lambda=1
use_augmented_data=False
use_lowercase_data=False
use_finetuning=True
use_validation_during_pretraining=True
use_translated_data=False
use_pretraining=False
load_model=".. /input/ver-22-parcor-output/21_06_2020_05_03_28_zero_shot_pro.h5"
use_english_validation=False
use_label_rounding=True
use_label_filtering=True
%matplotlib inline 
<drop_column>
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
data_undersampled=data.drop(data.query('Target == 4' ).sample(frac=.75 ).index )<prepare_x_and_y>
@tf.custom_gradient
def grad_reverse(x):
y = tf.identity(x)
 def custom_grad(dy):
return tf.math.scalar_mul(-1,dy)
 return y, custom_grad
class GradReverse(tf.keras.layers.Layer):
def __init__(self):
super().__init__()
def call(self, x):
return grad_reverse(x )
X=data_undersampled.drop(['Id', 'idhogar', 'Target', 'edjefe', 'edjefa'], axis=1)
 y=data_undersampled['Target']<split>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2 )<import_modules>
AUTO = tf.data.experimental.AUTOTUNE
if not localEnvironment and not IN_COLAB:
GCS_DS_PATH = KaggleDatasets().get_gcs_path("jigsaw-multilingual-toxic-comment-classification")
 NEXAMPLESPEREPOCH=240000
EPOCHS = 10
if strategy.num_replicas_in_sync==1:
BATCH_SIZE = 4 * strategy.num_replicas_in_sync
else:
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
if localEnvironment:
MODEL='jplu/tf-xlm-roberta-base'
if IN_COLAB:
MODEL='jplu/tf-xlm-roberta-base'
print("BATCH_SIZE: ", BATCH_SIZE )
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV<choose_model_class>
if IN_COLAB:
project_id="global-sun-279412"
clf = RandomForestClassifier()
params={'n_estimators': list(range(40,61, 1)) }
gs = GridSearchCV(clf, params, cv=5 )<train_model>
!gcloud config set project {project_id}
gs.fit(X_train, y_train )<predict_on_test>
bucket_name="jigsaw-tfrecords"
preds=gs.predict(X_test )<compute_test_metric>
if IN_COLAB:
auth.authenticate_user()
print(classification_report(y_test, preds))<compute_test_metric>
if localEnvironment:
root =".. "
elif IN_COLAB:
root='gs://jigsaw-tfrecords/'
else:
root="/kaggle"
root
print(confusion_matrix(y_test, preds))<find_best_params>
validation_fold_index=4
validation_fold_index
print(gs.best_params_)
 print(gs.best_score_)
 print(gs.best_estimator_ )<compute_test_metric>
def validationFoldRemoveFilter(validation_fold_index):
def _filter(features):
return tf.math.not_equal(features["ShardIndex"],validation_fold_index)
 return _filter
cvres = gs.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
print(np.sqrt(mean_score), params )<load_from_csv>
def validationFoldOnlyFilter(validation_fold_index):
def _filter(features):
return tf.math.equal(features["ShardIndex"],validation_fold_index)
 return _filter
test_data=pd.read_csv('.. /input/test.csv' )<drop_column>
if IN_COLAB:
TF_DS_PATH=root+"jigsaw-official"
elif not localEnvironment:
TF_DS_PATH = KaggleDatasets().get_gcs_path("jigsawaugmented")+"/tfrecords"
else:
TF_DS_PATH = ".. /tfrecords"
TF_DS_PATH
ids=test_data['Id']
test_data.drop(['Id', 'idhogar'], axis=1, inplace=True )<predict_on_test>
file="en1_train_non_toxic_PropercaseSentences_0_4.tfrecord" 
("Sentences" not in file)or("LowercaseSentences" in file )
test_predictions=gs.predict(test_data )<create_dataframe>
data_files=tf.io.gfile.glob(TF_DS_PATH + "/*.tfrecord")
 if use_augmented_data:
pass
elif use_lowercase_data:
data_files=list(filter(lambda file:("Sentences" not in file)or("LowercaseSentences" in file),data_files))
else:
data_files=list(filter(lambda file:"Sentences" not in file,data_files))
len(data_files),data_files
submit=pd.DataFrame({'Id': ids, 'Target': test_predictions} )<save_to_csv>
toxic_en_files=list(filter(lambda file:"nen" not in file and "toxic" in file and "non_toxic" not in file and "valid" not in file and "sharded" not in file, data_files))
toxic_en_files
submit.to_csv('submit.csv', index=False )<set_options>
non_toxic_en_files=list(filter(lambda file:"nen" not in file and "non_toxic" in file and "valid" not in file and "sharded" not in file, data_files))
non_toxic_en_files
sns.set(font_scale=2.2)
 plt.style.use('seaborn')
 warnings.filterwarnings('ignore')
<load_from_csv>
non_toxic_nen_files=list(filter(lambda file:"non_toxic" in file and "nen1" in file, data_files))
non_toxic_nen_files
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )<define_variables>
toxic_nen_files=list(filter(lambda file:"non_toxic" not in file and "nen1" in file, data_files))
toxic_nen_files
description = [ 
("v2a1"," Monthly rent payment"), 
("hacdor"," =1 Overcrowding by bedrooms"), 
("rooms"," number of all rooms in the house"), 
("hacapo"," =1 Overcrowding by rooms"), 
("v14a"," =1 has toilet in the household"), 
("refrig"," =1 if the household has refrigerator"), 
("v18q"," owns a tablet"), 
("v18q1"," number of tablets household owns"), 
("r4h1"," Males younger than 12 years of age"), 
("r4h2"," Males 12 years of age and older"), 
("r4h3"," Total males in the household"), 
("r4m1"," Females younger than 12 years of age"), 
("r4m2"," Females 12 years of age and older"), 
("r4m3"," Total females in the household"), 
("r4t1"," persons younger than 12 years of age"), 
("r4t2"," persons 12 years of age and older"), 
("r4t3"," Total persons in the household"), 
("tamhog"," size of the household"), 
("tamviv"," number of persons living in the household"), 
("escolari"," years of schooling"), 
("rez_esc"," Years behind in school"), 
("hhsize"," household size"), 
("paredblolad"," =1 if predominant material on the outside wall is block or brick"), 
("paredzocalo"," =1 if predominant material on the outside wall is socket(wood, zinc or absbesto"), 
("paredpreb"," =1 if predominant material on the outside wall is prefabricated or cement"), 
("pareddes"," =1 if predominant material on the outside wall is waste material"), 
("paredmad"," =1 if predominant material on the outside wall is wood"), 
("paredzinc"," =1 if predominant material on the outside wall is zink"), 
("paredfibras"," =1 if predominant material on the outside wall is natural fibers"), 
("paredother"," =1 if predominant material on the outside wall is other"), 
("pisomoscer"," =1 if predominant material on the floor is mosaic ceramic terrazo"), 
("pisocemento"," =1 if predominant material on the floor is cement"), 
("pisoother"," =1 if predominant material on the floor is other"), 
("pisonatur"," =1 if predominant material on the floor is natural material"), 
("pisonotiene"," =1 if no floor at the household"), 
("pisomadera"," =1 if predominant material on the floor is wood"), 
("techozinc"," =1 if predominant material on the roof is metal foil or zink"), 
("techoentrepiso"," =1 if predominant material on the roof is fiber cement, mezzanine "), 
("techocane"," =1 if predominant material on the roof is natural fibers"), 
("techootro"," =1 if predominant material on the roof is other"), 
("cielorazo"," =1 if the house has ceiling"), 
("abastaguadentro"," =1 if water provision inside the dwelling"), 
("abastaguafuera"," =1 if water provision outside the dwelling"), 
("abastaguano"," =1 if no water provision"), 
("public"," =1 electricity from CNFL, ICE, ESPH/JASEC"), 
("planpri"," =1 electricity from private plant"), 
("noelec"," =1 no electricity in the dwelling"), 
("coopele"," =1 electricity from cooperative"), 
("sanitario1"," =1 no toilet in the dwelling"), 
("sanitario2"," =1 toilet connected to sewer or cesspool"), 
("sanitario3"," =1 toilet connected to septic tank"), 
("sanitario5"," =1 toilet connected to black hole or letrine"), 
("sanitario6"," =1 toilet connected to other system"), 
("energcocinar1"," =1 no main source of energy used for cooking(no kitchen)"), 
("energcocinar2"," =1 main source of energy used for cooking electricity"), 
("energcocinar3"," =1 main source of energy used for cooking gas"), 
("energcocinar4"," =1 main source of energy used for cooking wood charcoal"), 
("elimbasu1"," =1 if rubbish disposal mainly by tanker truck"), 
("elimbasu2"," =1 if rubbish disposal mainly by botan hollow or buried"), 
("elimbasu3"," =1 if rubbish disposal mainly by burning"), 
("elimbasu4"," =1 if rubbish disposal mainly by throwing in an unoccupied space"), 
("elimbasu5"," =1 if rubbish disposal mainly by throwing in river, creek or sea"), 
("elimbasu6"," =1 if rubbish disposal mainly other"), 
("epared1"," =1 if walls are bad"), 
("epared2"," =1 if walls are regular"), 
("epared3"," =1 if walls are good"), 
("etecho1"," =1 if roof are bad"), 
("etecho2"," =1 if roof are regular"), 
("etecho3"," =1 if roof are good"), 
("eviv1"," =1 if floor are bad"), 
("eviv2"," =1 if floor are regular"), 
("eviv3"," =1 if floor are good"), 
("dis"," =1 if disable person"), 
("male"," =1 if male"), 
("female"," =1 if female"), 
("estadocivil1"," =1 if less than 10 years old"), 
("estadocivil2"," =1 if free or coupled uunion"), 
("estadocivil3"," =1 if married"), 
("estadocivil4"," =1 if divorced"), 
("estadocivil5"," =1 if separated"), 
("estadocivil6"," =1 if widow/er"), 
("estadocivil7"," =1 if single"), 
("parentesco1"," =1 if household head"), 
("parentesco2"," =1 if spouse/partner"), 
("parentesco3"," =1 if son/doughter"), 
("parentesco4"," =1 if stepson/doughter"), 
("parentesco5"," =1 if son/doughter in law"), 
("parentesco6"," =1 if grandson/doughter"), 
("parentesco7"," =1 if mother/father"), 
("parentesco8"," =1 if father/mother in law"), 
("parentesco9"," =1 if brother/sister"), 
("parentesco10"," =1 if brother/sister in law"), 
("parentesco11"," =1 if other family member"), 
("parentesco12"," =1 if other non family member"), 
("idhogar"," Household level identifier"), 
("hogar_nin"," Number of children 0 to 19 in household"), 
("hogar_adul"," Number of adults in household"), 
("hogar_mayor"," 
("hogar_total"," 
("dependency"," Dependency rate"), 
("edjefe"," years of education of male head of household"), 
("edjefa"," years of education of female head of household"), 
("meaneduc","average years of education for adults(18+)"), 
("instlevel1"," =1 no level of education"), 
("instlevel2"," =1 incomplete primary"), 
("instlevel3"," =1 complete primary"), 
("instlevel4"," =1 incomplete academic secondary level"), 
("instlevel5"," =1 complete academic secondary level"), 
("instlevel6"," =1 incomplete technical secondary level"), 
("instlevel7"," =1 complete technical secondary level"), 
("instlevel8"," =1 undergraduate and higher education"), 
("instlevel9"," =1 postgraduate higher education"), 
("bedrooms"," number of bedrooms"), 
("overcrowding"," 
("tipovivi1"," =1 own and fully paid house"), 
("tipovivi2"," =1 own, paying in installments"), 
("tipovivi3"," =1 rented"), 
("tipovivi4"," =1 precarious"), 
("tipovivi5"," =1 other(assigned"), 
("computer"," =1 if the household has notebook or desktop computer, borrowed)"), 
("television"," =1 if the household has TV"), 
("mobilephone"," =1 if mobile phone"), 
("qmobilephone"," 
("lugar1"," =1 region Central"), 
("lugar2"," =1 region Chorotega"), 
("lugar3"," =1 region PacÃƒÂ­fico central"), 
("lugar4"," =1 region Brunca"), 
("lugar5"," =1 region Huetar AtlÃƒÂ¡ntica"), 
("lugar6"," =1 region Huetar Norte"), 
("area1"," =1 zona urbana"), 
("area2"," =2 zona rural"), 
("age"," Age in years"), 
("SQBescolari"," escolari squared"), 
("SQBage"," age squared"), 
("SQBhogar_total"," hogar_total squared"), 
("SQBedjefe"," edjefe squared"), 
("SQBhogar_nin"," hogar_nin squared"), 
("SQBovercrowding"," overcrowding squared"), 
("SQBdependency"," dependency squared"), 
("SQBmeaned"," meaned squared"), 
("agesq"," Age squared"),]
description = pd.DataFrame(description, columns=['varname', 'description'] )<sort_values>
unknown_nen_files=list(filter(lambda file: "test" not in file and "nen2" in file, data_files))
unknown_nen_files
total = df_train.isnull().sum().sort_values(ascending=False)
 percent = 100 *(df_train.isnull().sum() / df_train.isnull().count() ).sort_values(ascending=False)
 missing_df = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_df.head(20 )<feature_engineering>
valid_files=list(filter(lambda file:"test" not in file and "nen1" in file and "Sentences" not in file, data_files))
valid_files
df_train.loc[(df_train['edjefa'] == "yes")&(df_train['parentesco1'] == 1), "edjefa"] = df_train.loc[(df_train['edjefa'] == "yes")&(df_train['parentesco1'] == 1), "escolari"]
df_train.loc[(df_train['edjefe'] == "yes")&(df_train['parentesco1'] == 1), "edjefe"] = df_train.loc[(df_train['edjefe'] == "yes")&(df_train['parentesco1'] == 1), "escolari"]
df_test.loc[(df_test['edjefa'] == "yes")&(df_test['parentesco1'] == 1), "edjefa"] = df_test.loc[(df_test['edjefa'] == "yes")&(df_test['parentesco1'] == 1), "escolari"]
df_test.loc[(df_test['edjefe'] == "yes")&(df_test['parentesco1'] == 1), "edjefe"] = df_test.loc[(df_test['edjefe'] == "yes")&(df_test['parentesco1'] == 1), "escolari"]
df_train.loc[df_train['edjefa'] == "yes", "edjefa"] = 4
df_train.loc[df_train['edjefe'] == "yes", "edjefe"] = 4
df_test.loc[df_test['edjefa'] == "yes", "edjefa"] = 4
df_test.loc[df_test['edjefe'] == "yes", "edjefe"] = 4
df_train['edjef'] = np.max(df_train[['edjefa','edjefe']], axis=1)
 df_test['edjef'] = np.max(df_test[['edjefa','edjefe']], axis=1)
 df_train.loc[(df_train.v14a == 1)&(df_train.sanitario1 == 1)&(df_train.abastaguano == 0), "v14a"] = 0
df_train.loc[(df_train.v14a == 1)&(df_train.sanitario1 == 1)&(df_train.abastaguano == 0), "sanitario1"] = 0
df_test.loc[(df_test.v14a == 1)&(df_test.sanitario1 == 1)&(df_test.abastaguano == 0), "v14a"] = 0
df_test.loc[(df_test.v14a == 1)&(df_test.sanitario1 == 1)&(df_test.abastaguano == 0), "sanitario1"] = 0<data_type_conversions>
if IN_COLAB:
TRANLSATED_DATA_PATH_ONE=root+"/translated-train-bias-all-langs"
TRANLSATED_DATA_PATH_TWO=root+"/jigsaw-train-multilingual-coments-google-api"
elif not localEnvironment:
TRANSLATED_DATA_PATH = KaggleDatasets().get_gcs_path("jigsawtranslated")+"/tfrecords-translated"
else:
pass
TRANSLATED_DATA_PATH
df_train['rez_esc'].fillna(0, inplace=True)
 df_test['rez_esc'].fillna(0, inplace=True )<data_type_conversions>
toxic_translated_files=list(filter(lambda file:"nen" not in file and "toxic" in file and "non" not in file and "valid" not in file and "sharded" not in file, translated_files))
random.choice(toxic_translated_files )
df_train['SQBmeaned'].fillna(0, inplace=True)
 df_test['SQBmeaned'].fillna(0, inplace=True )<data_type_conversions>
non_toxic_translated_files=list(filter(lambda file:"nen" not in file and "toxic" in file and "non" in file and "valid" not in file and "sharded" not in file, translated_files))
random.choice(non_toxic_translated_files )
df_train['meaneduc'].fillna(0, inplace=True)
 df_test['meaneduc'].fillna(0, inplace=True )<count_values>
df_train['v18q'].value_counts()<count_values>
opus_langs=[['en', 'it'],
['en', 'ru'],
['en', 'tr'],
['en', 'pt_br'],
['en', 'fr'],
['en', 'pt'],
['en', 'es'],
['en', 'hi']]
if IN_COLAB:
OPUS_DS_PATH=root+"opus"
elif not localEnvironment:
OPUS_DS_PATH = KaggleDatasets().get_gcs_path("opus-for-jigsaw-sharded")
 else:
OPUS_DS_PATH = ".. /opus-processed"
OPUS_DS_PATH
df_train.loc[df_train['v18q'] == 1, 'v18q1'].value_counts()<count_values>
if IN_COLAB:
extension=".gz"
else:
extension=""
opus_files=tf.io.gfile.glob(OPUS_DS_PATH + "/*.tfrecord"+extension)
 if interativeEnvironment:
print(opus_files)
df_train.loc[df_train['v18q'] == 0, 'v18q1'].value_counts()<data_type_conversions>
feature_description = {
'tokens': tf.io.FixedLenFeature([192], tf.int64),
'mask': tf.io.FixedLenFeature([192], tf.int64),
'toxic': tf.io.FixedLenFeature([], tf.float32),
's_toxic': tf.io.FixedLenFeature([], tf.float32),
'source':tf.io.FixedLenFeature([], tf.int64),
'toxicity_annotator_count':tf.io.FixedLenFeature([], tf.int64),
'lang':tf.io.FixedLenFeature([], tf.string),
'ShardIndex':tf.io.FixedLenFeature([], tf.int64),
}
def _parse_proto(example_proto):
example= tf.io.parse_single_example(example_proto, feature_description)
 return example
def _form_tuple(example):
y={"toxic":tf.cast(example['s_toxic'],tf.float32)}
if use_dann:
y["lang"]=tf.cast(tf.math.equal(example['lang'],'en'),tf.float32)
 if use_parcor:
y["is_same"]=example["is_same"]
return(example['tokens'],example['mask']),y
def smoother(smoothing=0.01):
def smooth(example):
label=tf.cast(example['s_toxic'],tf.float32)
 label=tf.cond(label>=1.0-smoothing,lambda :tf.ones_like(label)-smoothing,lambda :tf.cond(label<0.0+smoothing,lambda :tf.zeros_like(label)+smoothing,lambda :label))
example['s_toxic']=label
return example
return smooth
def isValidLabel(example):
example['isLabelValid']=tf.cond(example['toxic']==-1,lambda :tf.constant(0.0),lambda :tf.constant(1.0))
return example
def _filter_non_confident(example):
return(example["s_toxic"]>0.5)or(example["s_toxic"]<0.25 )
df_train['v18q1'].fillna(0, inplace=True)
 df_test['v18q1'].fillna(0, inplace=True )<count_values>
def roundLabels(example):
label=tf.cast(example['s_toxic'],tf.float32)
 label=tf.cond(label>=0.5,lambda :tf.ones_like(label),lambda :tf.zeros_like(label))
example['s_toxic']=label
return example
df_train['tipovivi3'].value_counts()<data_type_conversions>
def perFileDataset(filename):
tf.print("Retrieving from filename",filename)
 return tf.data.TFRecordDataset(filename ).map(_parse_proto)
 def perFileDatasetShuffled(filename):
return perFileDataset(filename ).shuffle(200000)
 def unsqueezeTargetDimensions(features,labels):
labels=tf.cast(labels,tf.float32)
 return features,tf.stack([tf.math.subtract(tf.constant([1.0]),labels),labels],axis=-1),
df_train['v2a1'].fillna(0, inplace=True)
 df_test['v2a1'].fillna(0, inplace=True )<sort_values>
def toxic_train_en_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(toxic_en_files ).shuffle(2048 ).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12 ).map(smoother())
 def toxic_train_translated_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(toxic_translated_files ).shuffle(2048*4 ).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12 ).map(smoother(0.1))
total = df_train.isnull().sum().sort_values(ascending=False)
 percent = 100 *(df_train.isnull().sum() / df_train.isnull().count() ).sort_values(ascending=False)
 missing_df = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_df.head(20 )<sort_values>
def non_toxic_train_en_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(non_toxic_en_files ).shuffle(2048 ).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12 ).map(smoother())
 def non_toxic_train_translated_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(non_toxic_translated_files ).shuffle(2048*4 ).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12 ).map(smoother(0.1))
total = df_test.isnull().sum().sort_values(ascending=False)
 percent = 100 *(df_test.isnull().sum() / df_test.isnull().count() ).sort_values(ascending=False)
 missing_df = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_df.head(20 )<define_variables>
def toxic_train_nen_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(toxic_nen_files ).shuffle(2048 ).interleave(perFileDataset ).filter(validationFoldRemoveFilter(validation_fold_index)).map(smoother() ).shuffle(8000)
features_object = [col for col in df_train.columns if df_train[col].dtype == 'object']<feature_engineering>
def non_toxic_train_nen_dataset() :
return tf.data.Dataset.from_tensor_slices(non_toxic_nen_files ).shuffle(2048 ).interleave(perFileDataset ).filter(validationFoldRemoveFilter(validation_fold_index)).map(smoother() ).shuffle(8000)
df_train['dependency'] = np.sqrt(df_train['SQBdependency'])
 df_test['dependency'] = np.sqrt(df_test['SQBdependency'] )<data_type_conversions>
def unknown_nen_dataset() :
return tf.data.Dataset.from_tensor_slices(unknown_nen_files ).shuffle(2048 ).interleave(perFileDataset,num_parallel_calls=AUTO)
<data_type_conversions>
def valid_nen_dataset() :
return tf.data.Dataset.from_tensor_slices(valid_files ).interleave(perFileDataset ).filter(validationFoldOnlyFilter(validation_fold_index))
def replace_edjefe(x):
if x == 'yes':
return 1
elif x == 'no':
return 0
else:
return x
df_train['edjefe'] = df_train['edjefe'].apply(replace_edjefe ).astype(float)
 df_test['edjefe'] = df_test['edjefe'].apply(replace_edjefe ).astype(float )<data_type_conversions>
def finetune_nen_dataset() :
return tf.data.Dataset.from_tensor_slices(valid_files ).shuffle(2048 ).interleave(perFileDataset ).filter(validationFoldRemoveFilter(validation_fold_index))
def replace_edjefa(x):
if x == 'yes':
return 1
elif x == 'no':
return 0
else:
return x
df_train['edjefa'] = df_train['edjefa'].apply(replace_edjefa ).astype(float)
 df_test['edjefa'] = df_test['edjefa'].apply(replace_edjefa ).astype(float )<feature_engineering>
%%time
if interativeEnvironment:
total_by_source=Counter()
total_by_toxic=Counter()
total=0
for i,l in tqdm(enumerate(non_toxic_train_nen_dataset())) :
total+=1
total_by_source.update([l["source"].numpy() ])
 total_by_toxic.update([l["s_toxic"].numpy() ])
 total_nen_train_non_toxic=total
total,total_by_toxic.most_common() ,total_by_source.most_common()
else:
total_nen_train_non_toxic=5956
df_train['edjef'] = np.max(df_train[['edjefa','edjefe']], axis=1)
 df_test['edjef'] = np.max(df_test[['edjefa','edjefe']], axis=1 )<feature_engineering>
%%time
if interativeEnvironment:
total_by_source=Counter()
total_by_toxic=Counter()
total=0
for i,l in tqdm(enumerate(toxic_train_nen_dataset())) :
total+=1
total_by_source.update([l["source"].numpy() ])
 total_by_toxic.update([l["s_toxic"].numpy() ])
 total_nen_train_toxic=total
total,total_by_toxic.most_common() ,total_by_source.most_common()
else:
total_nen_train_toxic=1061
df_train['roof_waste_material'] = np.nan
df_test['roof_waste_material'] = np.nan
df_train['electricity_other'] = np.nan
df_test['electricity_other'] = np.nan
def fill_roof_exception(x):
if(x['techozinc'] == 0)and(x['techoentrepiso'] == 0)and(x['techocane'] == 0)and(x['techootro'] == 0):
return 1
else:
return 0
def fill_no_electricity(x):
if(x['public'] == 0)and(x['planpri'] == 0)and(x['noelec'] == 0)and(x['coopele'] == 0):
return 1
else:
return 0
df_train['roof_waste_material'] = df_train.apply(lambda x : fill_roof_exception(x),axis=1)
 df_test['roof_waste_material'] = df_test.apply(lambda x : fill_roof_exception(x),axis=1)
 df_train['electricity_other'] = df_train.apply(lambda x : fill_no_electricity(x),axis=1)
 df_test['electricity_other'] = df_test.apply(lambda x : fill_no_electricity(x),axis=1 )<define_variables>
%%time
if interativeEnvironment:
total_by_source=Counter()
total_by_toxic=Counter()
total=0
for i,l in tqdm(enumerate(unknown_nen_dataset())) :
total+=1
total_by_source.update([l["source"].numpy() ])
 total_by_toxic.update([l["s_toxic"].numpy() ])
 total_nen_unknown=total
total,total_by_toxic.most_common() ,total_by_source.most_common()
else:
total_nen_unknown=71078
binary_cat_features = [col for col in df_train.columns if df_train[col].value_counts().shape[0] == 2]<define_variables>
n_steps = NEXAMPLESPEREPOCH // BATCH_SIZE
num_nen=0
if use_validation_during_pretraining:
num_nen+=total_nen_train_toxic+total_nen_train_non_toxic
if use_dann:
num_nen += total_nen_unknown
n_steps,num_nen
continuous_features = [col for col in df_train.columns if col not in binary_cat_features]
continuous_features = [col for col in continuous_features if col not in features_object]
continuous_features = [col for col in continuous_features if col not in ['Id', 'Target', 'idhogar']]<filter>
def applyOptions(ds):
if use_label_filtering:
ds=ds.filter(_filter_non_confident)
 if use_label_rounding:
ds=ds.map(roundLabels)
 ds=ds.map(smoother())
 return ds
num_dataset_section=2
if use_translated_data:
num_dataset_section+=2
def f0() :
return unknown_nen_dataset().repeat().take(total_nen_unknown)
 def f1() :
return applyOptions(toxic_train_en_dataset().repeat().take(int(( NEXAMPLESPEREPOCH-num_nen)/num_dataset_section)))
 def f2() :
return applyOptions(non_toxic_train_en_dataset().repeat().take(int(( NEXAMPLESPEREPOCH-num_nen)/num_dataset_section)))
 def f3() :
return applyOptions(toxic_train_translated_dataset().repeat().take(int(( NEXAMPLESPEREPOCH-num_nen)/num_dataset_section)))
 def f4() :
return applyOptions(non_toxic_train_translated_dataset().repeat().take(int(( NEXAMPLESPEREPOCH-num_nen)/num_dataset_section)))
 def f5() :
return toxic_train_nen_dataset()
def f6() :
return non_toxic_train_nen_dataset()
branches={0:f0,1:f1,2:f2,3:f3,4:f4,5:f5,6:f6}
mixedDatasetParams=[1,2]
if use_dann:
mixedDatasetParams.append(0)
 if use_validation_during_pretraining:
mixedDatasetParams.extend([5,6])
 if use_translated_data:
mixedDatasetParams.extend([3,4])
 def mapParamsToDataset(params):
return tf.switch_case(params,branches)
print('There are {} continuous features'.format(len(continuous_features)))
 for col in continuous_features:
print('{}: {}'.format(col, description.loc[description['varname'] == col, 'description'].values))<count_values>
cycle_length=len(mixedDatasetParams)
 def non_parallel_dataset(params=mixedDatasetParams):
print(params)
 return tf.data.Dataset.from_tensor_slices(params ).interleave(mapParamsToDataset, num_parallel_calls=AUTO,cycle_length=cycle_length ).take(NEXAMPLESPEREPOCH)
 if interativeEnvironment:
train_dict_dataset=non_parallel_dataset()
train_dict_dataset
df_train['edjef'].value_counts()<drop_column>
non_parallel_dataset()
df_train.drop('tamhog', axis=1, inplace=True)
 df_test.drop('tamhog', axis=1, inplace=True )<feature_engineering>
a=[1,2,3,4]
b=[1,2,3,4]
a==b
df_train['adult'] = df_train['hogar_adul'] - df_train['hogar_mayor']
df_train['dependency_count'] = df_train['hogar_nin'] + df_train['hogar_mayor']
df_train['dependency'] = df_train['dependency_count'] / df_train['adult']
df_train['child_percent'] = df_train['hogar_nin'] / df_train['hogar_total']
df_train['elder_percent'] = df_train['hogar_mayor'] / df_train['hogar_total']
df_train['adult_percent'] = df_train['hogar_adul'] / df_train['hogar_total']
df_train['males_younger_12_years_percent'] = df_train['r4h1'] / df_train['hogar_total']
df_train['males_older_12_years_percent'] = df_train['r4h2'] / df_train['hogar_total']
df_train['males_percent'] = df_train['r4h3'] / df_train['hogar_total']
df_train['females_younger_12_years_percent'] = df_train['r4m1'] / df_train['hogar_total']
df_train['females_older_12_years_percent'] = df_train['r4m2'] / df_train['hogar_total']
df_train['females_percent'] = df_train['r4m3'] / df_train['hogar_total']
df_train['persons_younger_12_years_percent'] = df_train['r4t1'] / df_train['hogar_total']
df_train['persons_older_12_years_percent'] = df_train['r4t2'] / df_train['hogar_total']
df_train['persons_percent'] = df_train['r4t3'] / df_train['hogar_total']<feature_engineering>
%%time
if interativeEnvironment:
dlist=non_parallel_dataset().shuffle(NEXAMPLESPEREPOCH ).repeat()
epoch_metrics={}
shard_metrics={}
signature_examples_source={}
signature_examples_count={}
for i,l in tqdm(enumerate(dlist)) :
epoch=str(i//NEXAMPLESPEREPOCH)
 shard=str(i//40000)
 if epoch=="20":
break
if epoch in epoch_metrics.keys() :
pass
else:
total_by_source=Counter()
total_by_toxic=Counter()
total_by_toxic_source=Counter()
total_by_s_toxic=Counter()
total=0
epoch_metrics[epoch]={"by_toxic":total_by_toxic,"by_source":total_by_source,"total":total,"by_both":total_by_toxic_source,"by_s_toxic":total_by_s_toxic}
if shard in shard_metrics.keys() :
pass
else:
stotal_by_source=Counter()
stotal_by_toxic=Counter()
stotal_by_toxic_source=Counter()
stotal_by_s_toxic=Counter()
stotal=0
shard_metrics[shard]={"by_toxic":stotal_by_toxic,"by_source":stotal_by_source,"total":stotal,"by_both":stotal_by_toxic_source,"by_s_toxic":stotal_by_s_toxic}
total+=1
if l["source"].numpy() not in signature_examples_source:
signature_examples_source[l["source"].numpy() ]=l
signature_examples_count[l["source"].numpy() ]=0
if all(signature_examples_source[l["source"].numpy() ]["tokens"].numpy() ==l["tokens"].numpy()):
signature_examples_count[l["source"].numpy() ]+=1
total_by_source.update([l["source"].numpy() ])
 total_by_toxic.update([-1 if l["toxic"] ==-1 else 1 if l["toxic"].numpy() >=0.5 else 0])
 total_by_s_toxic.update([-1 if l["s_toxic"] ==-1 else 1 if l["s_toxic"].numpy() >=0.5 else 0])
 total_by_toxic_source.update([(round(l["toxic"].numpy()),l["source"].numpy())])
 stotal+=1
stotal_by_source.update([l["source"].numpy() ])
 stotal_by_toxic.update([-1 if l["toxic"] ==-1 else 1 if l["toxic"].numpy() >=0.5 else 0])
 stotal_by_s_toxic.update([-1 if l["s_toxic"] ==-1 else 1 if l["s_toxic"].numpy() >=0.5 else 0])
 stotal_by_toxic_source.update([(round(l["toxic"].numpy()),l["source"].numpy())])
 for epoch,epoch_metric in epoch_metrics.items() :
print("Epoch:-",epoch,"Toxic:-",epoch_metric["by_toxic"].most_common() ,"By SToxic:",epoch_metric["by_s_toxic"].most_common() ,"By source:-",epoch_metric["by_source"].most_common() ,"
")
 print(signature_examples_count )
df_test['adult'] = df_test['hogar_adul'] - df_test['hogar_mayor']
df_test['dependency_count'] = df_test['hogar_nin'] + df_test['hogar_mayor']
df_test['dependency'] = df_test['dependency_count'] / df_test['adult']
df_test['child_percent'] = df_test['hogar_nin'] / df_test['hogar_total']
df_test['elder_percent'] = df_test['hogar_mayor'] / df_test['hogar_total']
df_test['adult_percent'] = df_test['hogar_adul'] / df_test['hogar_total']
df_test['males_younger_12_years_percent'] = df_test['r4h1'] / df_test['hogar_total']
df_test['males_older_12_years_percent'] = df_test['r4h2'] / df_test['hogar_total']
df_test['males_percent'] = df_test['r4h3'] / df_test['hogar_total']
df_test['females_younger_12_years_percent'] = df_test['r4m1'] / df_test['hogar_total']
df_test['females_older_12_years_percent'] = df_test['r4m2'] / df_test['hogar_total']
df_test['females_percent'] = df_test['r4m3'] / df_test['hogar_total']
df_test['persons_younger_12_years_percent'] = df_test['r4t1'] / df_test['hogar_total']
df_test['persons_older_12_years_percent'] = df_test['r4t2'] / df_test['hogar_total']
df_test['persons_percent'] = df_test['r4t3'] / df_test['hogar_total']<feature_engineering>
if interativeEnvironment:
for epoch,epoch_metric in shard_metrics.items() :
print("Shard:-",epoch,"Toxic:-",epoch_metric["by_toxic"].most_common() ,"Stoxic:",epoch_metric["by_s_toxic"].most_common() ,"By source:-",epoch_metric["by_source"].most_common() ,"
" )
df_train['males_younger_12_years_in_household_size'] = df_train['r4h1'] / df_train['hhsize']
df_train['males_older_12_years_in_household_size'] = df_train['r4h2'] / df_train['hhsize']
df_train['males_in_household_size'] = df_train['r4h3'] / df_train['hhsize']
df_train['females_younger_12_years_in_household_size'] = df_train['r4m1'] / df_train['hhsize']
df_train['females_older_12_years_in_household_size'] = df_train['r4m2'] / df_train['hhsize']
df_train['females_in_household_size'] = df_train['r4m3'] / df_train['hogar_total']
df_train['persons_younger_12_years_in_household_size'] = df_train['r4t1'] / df_train['hhsize']
df_train['persons_older_12_years_in_household_size'] = df_train['r4t2'] / df_train['hhsize']
df_train['persons_in_household_size'] = df_train['r4t3'] / df_train['hhsize']<feature_engineering>
def opus_files(lang):
return tf.io.gfile.glob(OPUS_DS_PATH + "/{}_*.tfrecord".format(lang)+extension)
 opus_files("es" )
df_test['males_younger_12_years_in_household_size'] = df_test['r4h1'] / df_test['hhsize']
df_test['males_older_12_years_in_household_size'] = df_test['r4h2'] / df_test['hhsize']
df_test['males_in_household_size'] = df_test['r4h3'] / df_test['hhsize']
df_test['females_younger_12_years_in_household_size'] = df_test['r4m1'] / df_test['hhsize']
df_test['females_older_12_years_in_household_size'] = df_test['r4m2'] / df_test['hhsize']
df_test['females_in_household_size'] = df_test['r4m3'] / df_test['hogar_total']
df_test['persons_younger_12_years_in_household_size'] = df_test['r4t1'] / df_test['hhsize']
df_test['persons_older_12_years_in_household_size'] = df_test['r4t2'] / df_test['hhsize']
df_test['persons_in_household_size'] = df_test['r4t3'] / df_test['hhsize']<feature_engineering>
if use_parcor:
parallel_feature_description = {
'tokens_en': tf.io.FixedLenFeature([192], tf.int64),
'en_attn_mask': tf.io.FixedLenFeature([192], tf.int64),
'tokens_nen': tf.io.FixedLenFeature([192], tf.int64),
'nen_attn_mask': tf.io.FixedLenFeature([192], tf.int64),
'lang':tf.io.FixedLenFeature([], tf.string),
'source':tf.io.FixedLenFeature([], tf.int64,default_value=[5]),
's_toxic':tf.io.FixedLenFeature([2], tf.float32,default_value=[-1.0,-1.0]),
'is_same':tf.io.FixedLenFeature([], tf.float32,default_value=[1.0]),
}
def _parse_parallel_proto(example_proto):
example= tf.io.parse_single_example(example_proto, parallel_feature_description)
 return example
if IN_COLAB:
compression_type="GZIP"
else:
compression_type=""
def perFileParallelDataset(filename,feature_description=feature_description):
tf.print("Retrieving from filename",filename)
 return tf.data.TFRecordDataset(filename,compression_type=compression_type ).map(_parse_parallel_proto)
 def make_parallel_pairs(example):
ret={}
ret["tokens"]=tf.stack([example["tokens_en"],example["tokens_nen"]])
 ret["mask"]=tf.stack([example["en_attn_mask"],example["nen_attn_mask"]])
 ret["s_toxic"]=example["s_toxic"]
ret["is_same"]=example["is_same"]
ret["lang"]=example["lang"]
return ret
def parallel_dataset(files):
return tf.data.Dataset.from_tensor_slices(files ).shuffle(2048 ).interleave(perFileParallelDataset ).map(make_parallel_pairs)
 def parallel_labelled_dataset(labelled_dataset):
return labelled_dataset.batch(2 ).map(make_parallel_labelled_pairs)
 def make_parallel_labelled_pairs(example):
example["is_same"]=tf.constant(0.0)
 return example
langs=["es","fr","pt","tr","ru","it"]
langFiles=list(map(opus_files,langs))
def all_parallel_dataset() :
return tf.data.Dataset.from_tensor_slices(langFiles ).interleave(parallel_dataset, num_parallel_calls=AUTO,cycle_length=len(langs))
def parallel_nonparallel_mixer(params):
return tf.cond(params=="parallel",lambda :all_parallel_dataset().take(int(NEXAMPLESPEREPOCH/4)).map(_form_tuple),lambda :parallel_labelled_dataset(non_parallel_dataset() ).take(int(NEXAMPLESPEREPOCH/4)).map(_form_tuple))
def parcor_dataset() :
return tf.data.Dataset.from_tensor_slices(["parallel","nonparallel"] ).interleave(parallel_nonparallel_mixer,num_parallel_calls=2 )
df_train['overcrowding_room_and_bedroom'] =(df_train['hacdor'] + df_train['hacapo'])/2
df_test['overcrowding_room_and_bedroom'] =(df_test['hacdor'] + df_test['hacapo'])/2<feature_engineering>
if interativeEnvironment:
ds=list(parallel_labelled_dataset(valid_nen_dataset() ).take(7))
print(list(map(lambda a:a["is_same"],ds)))
df_train['escolari_age'] = df_train['escolari']/df_train['age']
df_test['escolari_age'] = df_test['escolari']/df_test['age']
df_train['age_12_19'] = df_train['hogar_nin'] - df_train['r4t1']
df_test['age_12_19'] = df_test['hogar_nin'] - df_test['r4t1']<feature_engineering>
valid_en_files=list(filter(lambda file:"valid" in file and "nen" not in file and "Sentences" not in file, data_files))
valid_en_files
df_train['phones-per-capita'] = df_train['qmobilephone'] / df_train['tamviv']
df_train['tablets-per-capita'] = df_train['v18q1'] / df_train['tamviv']
df_train['rooms-per-capita'] = df_train['rooms'] / df_train['tamviv']
df_train['rent-per-capita'] = df_train['v2a1'] / df_train['tamviv']<feature_engineering>
def valid_en_dataset() :
return tf.data.Dataset.from_tensor_slices(valid_en_files ).interleave(perFileDataset ).shuffle(200000 ).map(roundLabels )
df_test['phones-per-capita'] = df_test['qmobilephone'] / df_test['tamviv']
df_test['tablets-per-capita'] = df_test['v18q1'] / df_test['tamviv']
df_test['rooms-per-capita'] = df_test['rooms'] / df_test['tamviv']
df_test['rent-per-capita'] = df_test['v2a1'] / df_test['tamviv']<feature_engineering>
if use_parcor:
train_dataset=parcor_dataset().shuffle(NEXAMPLESPEREPOCH ).repeat().batch(int(BATCH_SIZE/2)).prefetch(AUTO)
 valid_dataset=parallel_labelled_dataset(valid_nen_dataset() ).map(_form_tuple ).batch(int(BATCH_SIZE/2)).prefetch(AUTO)
 finetune_dataset=parallel_labelled_dataset(finetune_nen_dataset() ).map(_form_tuple ).batch(int(BATCH_SIZE/2)).prefetch(AUTO)
 else:
if use_english_validation:
valid_dataset = valid_en_dataset().batch(BATCH_SIZE ).map(_form_tuple ).prefetch(AUTO)
 else:
valid_dataset = valid_nen_dataset().map(_form_tuple ).batch(BATCH_SIZE ).prefetch(AUTO)
 train_dataset=non_parallel_dataset().shuffle(NEXAMPLESPEREPOCH ).map(_form_tuple ).repeat().batch(BATCH_SIZE ).prefetch(AUTO)
 finetune_dataset=finetune_nen_dataset().shuffle(NEXAMPLESPEREPOCH ).map(_form_tuple ).batch(BATCH_SIZE ).prefetch(AUTO)
 valid_dataset
family_size_features = ['adult', 'hogar_adul', 'hogar_mayor', 'hogar_nin', 'hogar_total', 'r4h1',
'r4h2', 'r4h3', 'r4m1', 'r4m2', 'r4m3', 'r4t1', 'r4t2', 'r4t3', 'hhsize']
new_feats = []
for col in family_size_features:
new_col_name = 'new_{}_per_{}'.format('v2a1', col)
 new_feats.append(new_col_name)
 df_train[new_col_name] = df_train['v2a1'] / df_train[col]
df_test[new_col_name] = df_test['v2a1'] / df_test[col]<data_type_conversions>
class WeightedBinaryCrossentropy(tf.keras.losses.BinaryCrossentropy):
def __init__(self,
from_logits=False,
label_smoothing=0,
reduction=losses_utils.ReductionV2.AUTO,lossWeight=0.1,
name='binary_crossentropy'):

 super(WeightedBinaryCrossentropy, self ).__init__(
 name=name,
reduction=reduction,
from_logits=from_logits,
label_smoothing=label_smoothing)
 self.lossWeight=lossWeight
def call(self, y_true, y_pred):
l= tf.math.scalar_mul(self.lossWeight,super().call(y_true,y_pred))
return l
for col in new_feats:
df_train[col].replace([np.inf], np.nan, inplace=True)
 df_train[col].fillna(0, inplace=True)
 df_test[col].replace([np.inf], np.nan, inplace=True)
 df_test[col].fillna(0, inplace=True )<feature_engineering>
class MaskedBinaryCrossentropy(tf.keras.losses.BinaryCrossentropy):
def __init__(self,
from_logits=False,
label_smoothing=0,
reduction=losses_utils.ReductionV2.AUTO,
name='binary_crossentropy'):

 super(MaskedBinaryCrossentropy, self ).__init__(
 name=name,
reduction=reduction,
from_logits=from_logits,
label_smoothing=label_smoothing)
 def call(self, y_true, y_pred):
if use_parcor:
raise("Unimplemented")
 y_true=tf.reshape(y_true,[-1,1])
 y_pred=tf.reshape(y_pred,[-1,1])
 mask=tf.math.greater_equal(y_true,0)
 mask=tf.cast(mask,y_true.dtype)
 mask=tf.squeeze(mask,axis =-1)
 masked_y_true=mask * y_true
l=super().call(masked_y_true,y_pred)
 l=mask*l
if use_parcor:
l=tf.reshape(l,[-1,2])
 l=l[:,0]+l[:,1]
return l
new_feats = []
for col in family_size_features:
new_col_name = 'new_{}_per_{}'.format('rooms', col)
 new_feats.append(new_col_name)
 df_train[new_col_name] = df_train['rooms'] / df_train[col]
df_test[new_col_name] = df_test['rooms'] / df_test[col]
for col in new_feats:
df_train[col].replace([np.inf], np.nan, inplace=True)
 df_train[col].fillna(0, inplace=True)
 df_test[col].replace([np.inf], np.nan, inplace=True)
 df_test[col].fillna(0, inplace=True )<feature_engineering>
class SquareLoss(tf.keras.losses.Loss):
def __init__(self,reduction=losses_utils.ReductionV2.AUTO, name="squareLoss",lossWeight=0):
super().__init__(reduction=reduction,name=name)
 self.lossWeight=lossWeight
def call(self, y_true, y_pred):
is_same_masked= y_pred*y_true
loss=tf.math.scalar_mul(self.lossWeight,is_same_masked*is_same_masked)
 return loss
new_feats = []
for col in family_size_features:
new_col_name = 'new_{}_per_{}'.format('bedrooms', col)
 new_feats.append(new_col_name)
 df_train[new_col_name] = df_train['bedrooms'] / df_train[col]
df_test[new_col_name] = df_test['bedrooms'] / df_test[col]
for col in new_feats:
df_train[col].replace([np.inf], np.nan, inplace=True)
 df_train[col].fillna(0, inplace=True)
 df_test[col].replace([np.inf], np.nan, inplace=True)
 df_test[col].fillna(0, inplace=True )<feature_engineering>
def build_model(transformer, max_len=512):
if use_parcor:
pair_input_word_ids = Input(shape=(2,max_len,), dtype=tf.int32, name="input_word_ids")
 pair_attention_masks = Input(shape=(2,max_len,), dtype=tf.int32, name="attention_mask")
 input_word_ids = tf.keras.backend.reshape(pair_input_word_ids,shape=(-1,max_len))
attention_masks= tf.keras.backend.reshape(pair_attention_masks,shape=(-1,max_len))
inputs=[pair_input_word_ids,pair_attention_masks]
else:
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 attention_masks = Input(shape=(max_len,), dtype=tf.int32, name="attention_mask")
 inputs=[input_word_ids,attention_masks]
transformer_output = transformer(( input_word_ids,attention_masks))
sequence_output = transformer_output[0]
transformer_output
if pooling_mode == pooling_mode_cls_token:
transformed_features = sequence_output[:, 0, :]
cls_token=transformed_features
else:
float_mask=tf.cast(tf.expand_dims(attention_masks,axis=-1),tf.float32)
 lengths=tf.reduce_sum(float_mask,axis=-2)
 masked_vals=tf.multiply(sequence_output,float_mask)
 sums=tf.reduce_sum(masked_vals,axis=-2)
 avgs=tf.math.divide(sums,lengths)
 maxes=tf.math.reduce_max(masked_vals,axis=-2)
 transformed_features=tf.concat([avgs,maxes],axis=-1)
 cls_token=transformed_features
for dims in fc_dims:
cls_token=Dense(dims, activation='relu' )(cls_token)
 if use_dann:
lang_token=GradReverse()(transformed_features)
 for dims in fc_dims:
lang_token=Dense(dims, activation='relu' )(lang_token)
 lang=Dense(1, activation="sigmoid",name="lang" )(lang_token)
 if use_parcor:
toxic = Dense(1, activation="sigmoid",name="unpair_toxic" )(cls_token)
 toxic=tf.keras.backend.reshape(toxic,shape=(-1,2,1))
toxic=tf.keras.layers.Layer(name="toxic" )(toxic)
 is_same=tf.keras.layers.Subtract(name="is_same" )([toxic[:,1,:],toxic[:,0,:]])
 else:
toxic = Dense(1, activation="sigmoid",name="toxic" )(cls_token)
 out=[toxic]
if use_parcor:
out.append(is_same)
 celoss=MaskedBinaryCrossentropy(
 from_logits=False, label_smoothing=0, reduction=losses_utils.ReductionV2.AUTO,
name='crossentropy_loss' 
)
 loss={"toxic":celoss}
if use_dann:
domainGap=WeightedBinaryCrossentropy(
 from_logits=False, label_smoothing=0, reduction=losses_utils.ReductionV2.AUTO,
name='domaingap_loss',lossWeight=dann_lambda 
)
 out.append(lang)
 loss["lang"]=domainGap
if use_parcor:
loss["is_same"]=SquareLoss(lossWeight=parcor_lambda)
 model = Model(inputs=inputs, outputs=out)
 model.compile(Adam(lr=5e-6), loss=loss, metrics={"toxic":['accuracy',tf.keras.metrics.AUC(name="auc"),tf.keras.metrics.AUC(name="pr-auc",curve="PR")]})
 return model
new_feats = []
for col in family_size_features:
new_col_name = 'new_{}_per_{}'.format('v18q1', col)
 new_feats.append(new_col_name)
 df_train[new_col_name] = df_train['v18q1'] / df_train[col]
df_test[new_col_name] = df_test['v18q1'] / df_test[col]
for col in new_feats:
df_train[col].replace([np.inf], np.nan, inplace=True)
 df_train[col].fillna(0, inplace=True)
 df_test[col].replace([np.inf], np.nan, inplace=True)
 df_test[col].fillna(0, inplace=True )<feature_engineering>
def setTrainableLayers(transformer,embeddingsTrainable=False,poolerTrainable=False,hiddenTrainable=8):
transformer.layers[0].pooler.trainable=poolerTrainable
transformer.layers[0].embeddings.trainable=embeddingsTrainable
trainableHiddens=transformer.layers[0].encoder.layer[-1*hiddenTrainable:]
for layer in transformer.layers[0].encoder.layer:
if layer in trainableHiddens:
layer.trainable=True
else:
layer.trainable=False
new_feats = []
for col in family_size_features:
new_col_name = 'new_{}_per_{}'.format('qmobilephone', col)
 new_feats.append(new_col_name)
 df_train[new_col_name] = df_train['qmobilephone'] / df_train[col]
df_test[new_col_name] = df_test['qmobilephone'] / df_test[col]
for col in new_feats:
df_train[col].replace([np.inf], np.nan, inplace=True)
 df_train[col].fillna(0, inplace=True)
 df_test[col].replace([np.inf], np.nan, inplace=True)
 df_test[col].fillna(0, inplace=True )<feature_engineering>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 if load_model is not None:
model.load_weights(load_model)
 model.summary()
new_feats = []
for col in family_size_features:
new_col_name = 'new_{}_per_{}'.format('rez_esc', col)
 new_feats.append(new_col_name)
 df_train[new_col_name] = df_train['rez_esc'] / df_train[col]
df_test[new_col_name] = df_test['rez_esc'] / df_test[col]
for col in new_feats:
df_train[col].replace([np.inf], np.nan, inplace=True)
 df_train[col].fillna(0, inplace=True)
 df_test[col].replace([np.inf], np.nan, inplace=True)
 df_test[col].fillna(0, inplace=True )<feature_engineering>
now = datetime.now()
dt_string = now.strftime("%d_%m_%Y_%H_%M_%S_")
 out_path=dt_string+'zero_shot_pro.h5'
write_root=""
if IN_COLAB:
write_root=root+"outputs/"
out_path=write_root+out_path
print(out_path)
df_train['rez_esc_age'] = df_train['rez_esc'] / df_train['age']
df_train['rez_esc_escolari'] = df_train['rez_esc'] / df_train['escolari']
df_test['rez_esc_age'] = df_test['rez_esc'] / df_test['age']
df_test['rez_esc_escolari'] = df_test['rez_esc'] / df_test['escolari']<feature_engineering>
if use_pretraining:
metric_prefix=""
if use_parcor or use_dann:
metric_prefix="_toxic"
mc = ModelCheckpoint(out_path, monitor="val"+metric_prefix+"_auc", mode='max',verbose=1,save_weights_only=True,save_best_only=True)
 es = EarlyStopping(monitor="val"+metric_prefix+"_auc", mode='max',patience=2,verbose=1,restore_best_weights=True)
 lrScheduler=ReduceLROnPlateau(
 monitor="val"+metric_prefix+"_loss",
factor=0.2,
patience=1,
verbose=1,
mode="min",
min_delta=0,
cooldown=0,
min_lr=1e-10,
)
df_train['tabulet_x_qmobilephone'] = df_train['v18q1'] * df_train['qmobilephone']
df_test['tabulet_x_qmobilephone'] = df_test['v18q1'] * df_test['qmobilephone']<feature_engineering>
if use_pretraining:
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
verbose=1,
callbacks=[mc,es,lrScheduler]
)
for col1 in ['epared1', 'epared2', 'epared3']:
for col2 in ['etecho1', 'etecho2', 'etecho3']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]
for col1 in ['epared1', 'epared2', 'epared3']:
for col2 in ['eviv1', 'eviv2', 'eviv3']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]
for col1 in ['etecho1', 'etecho2', 'etecho3']:
for col2 in ['eviv1', 'eviv2', 'eviv3']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]<feature_engineering>
%%time
sub = pd.read_csv(root+'/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv',nrows=nrows )
for col1 in ['epared1', 'epared2', 'epared3']:
for col2 in ['etecho1', 'etecho2', 'etecho3']:
for col3 in ['eviv1', 'eviv2', 'eviv3']:
new_col_name = 'new_{}_x_{}_x_{}'.format(col1, col2, col3)
 df_train[new_col_name] = df_train[col1] * df_train[col2] * df_train[col3]
df_test[new_col_name] = df_test[col1] * df_test[col2] * df_train[col3]<feature_engineering>
test_files=list(filter(lambda file:"test" in file and "nen" in file, data_files))
test_files
for col1 in ['public', 'planpri', 'noelec', 'coopele']:
for col2 in ['energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]<feature_engineering>
if use_parcor:
test_dataset=parallel_labelled_dataset(tf.data.Dataset.from_tensor_slices(test_files ).interleave(perFileDataset,cycle_length=1)).map(_form_tuple ).batch(int(BATCH_SIZE/2)).prefetch(AUTO)
 else:
test_dataset=tf.data.Dataset.from_tensor_slices(test_files ).interleave(perFileDataset,cycle_length=1 ).batch(BATCH_SIZE ).map(_form_tuple ).prefetch(AUTO)
for col1 in ['sanitario1', 'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6']:
for col2 in ['elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4', 'elimbasu5', 'elimbasu6']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]<feature_engineering>
if use_pretraining:
l=model.predict(test_dataset, verbose=1)
 if use_dann:
l=l[0]
if use_parcor:
l=np.reshape(l,(-1,1))
sub['toxic'] = l[:,-1]
if use_finetuning:
write_file=write_root+'submission.unfinetuned.csv'
else:
write_file=write_root+'submission.csv'
sub.to_csv(write_file, index=False )
for col1 in ['abastaguadentro', 'abastaguafuera', 'abastaguano']:
for col2 in ['sanitario1', 'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]<feature_engineering>
if use_finetuning:
now = datetime.now()
dt_string = now.strftime("%d_%m_%Y_%H_%M_%S_")
 out_path=dt_string+'zero_shot_pro.h5'
write_root=""
if IN_COLAB:
write_root=root+"outputs/"
out_path=write_root+out_path
print(out_path)
for col1 in ['area1', 'area2']:
for col2 in ['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]<feature_engineering>
if use_finetuning:
metric_prefix=""
if use_parcor or use_dann:
metric_prefix="_toxic"
mc = ModelCheckpoint(out_path, monitor="val"+metric_prefix+"_auc", mode='max',verbose=1,save_weights_only=True,save_best_only=True)
 es = EarlyStopping(monitor="val"+metric_prefix+"_auc", mode='max',patience=2,verbose=1,restore_best_weights=True)
 lrScheduler=ReduceLROnPlateau(
 monitor="val"+metric_prefix+"_loss",
factor=0.2,
patience=1,
verbose=1,
mode="min",
min_delta=0,
cooldown=0,
min_lr=1e-10,
)
for col1 in ['lugar1', 'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6']:
for col2 in ['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']:
new_col_name = 'new_{}_x_{}'.format(col1, col2)
 df_train[new_col_name] = df_train[col1] * df_train[col2]
df_test[new_col_name] = df_test[col1] * df_test[col2]<feature_engineering>
if use_finetuning:
train_history = model.fit(
 finetune_dataset,
validation_data=valid_dataset,
epochs=EPOCHS,
verbose=1,
callbacks=[mc,es,lrScheduler]
)
df_train['electronics'] = df_train['computer'] * df_train['mobilephone'] * df_train['television'] * df_train['v18q'] * df_train['refrig']
df_test['electronics'] = df_test['computer'] * df_test['mobilephone'] * df_test['television'] * df_test['v18q'] * df_test['refrig']
df_train['no_appliances'] = df_train['refrig'] + df_train['computer'] + df_train['television'] + df_train['mobilephone']
df_test['no_appliances'] = df_test['refrig'] + df_test['computer'] + df_test['television'] + df_test['mobilephone']<feature_engineering>
if use_finetuning:
l=model.predict(test_dataset, verbose=1)
 if use_dann:
l=l[0]
if use_parcor:
l=np.reshape(l,(-1,1))
sub['toxic'] = l[:,-1]
write_file=write_root+'submission.csv'
sub.to_csv(write_file, index=False )
<count_values><EOS> 
def reset_keras() :
tf.keras.backend.clear_session()
print(gc.collect() )