Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<split>
import os, re
import tensorflow as tf
import pandas as pd
import numpy as np
import tensorflow as tf
import tensorflow.keras.layers as L
import tensorflow.keras.optimizers as Optim
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.15, random_state=12345 )<set_options>
def fast_encode(texts, tokenizer, chunk_size = 256, maxlen = 512):
tokenizer.enable_truncation(max_lenght = maxlen)
 tokenizer.enable_padding(max_length = maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tocanizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
class CRPSCallback(Callback):
def __init__(self,validation, predict_batch_size=20, include_on_batch=False):
super(CRPSCallback, self ).__init__()
self.validation = validation
self.predict_batch_size = predict_batch_size
self.include_on_batch = include_on_batch
print('validation shape',len(self.validation))
def on_batch_begin(self, batch, logs={}):
pass
def on_train_begin(self, logs={}):
if not('CRPS_score_val' in self.params['metrics']):
self.params['metrics'].append('CRPS_score_val')
 def on_batch_end(self, batch, logs={}):
if(self.include_on_batch):
logs['CRPS_score_val'] = float('-inf')
 def on_epoch_end(self, epoch, logs={}):
logs['CRPS_score_val'] = float('-inf')
 if(self.validation):
X_valid, y_valid = self.validation[0], self.validation[1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_valid.shape[0])
 val_s = np.round(val_s, 6)
 logs['CRPS_score_val'] = val_s 
<choose_model_class>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'])
def get_model(x_tr,y_tr,x_val,y_val):
inp = Input(shape =(x_tr.shape[1],))
x = Dense(1024, input_dim=X.shape[1], activation='relu' )(inp)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(512, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(256, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inp,out)
 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[])
 es = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=1,
patience=10)
 mc = ModelCheckpoint('best_model.h5',monitor='CRPS_score_val',mode='min',
save_best_only=True, verbose=1, save_weights_only=True)
 bsz = 1024
steps = x_tr.shape[0]/bsz
model.fit(x_tr, y_tr,callbacks=[CRPSCallback(validation =(x_val,y_val)) ,es,mc], epochs=100, batch_size=bsz,verbose=1)
 model.load_weights("best_model.h5")
 y_pred = model.predict(x_val)
 y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * x_val.shape[0])
 crps = np.round(val_s, 6)
 return model,crps<split>
def build_model(transformer, max_len = 512):
input_word_ids = L.Input(shape =(max_len,), dtype = tf.int32, name = 'input_word_ids')
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = L.Dense(1, activation = 'sigmoid' )(cls_token)
 model = Model(inputs = input_word_ids, outputs = out)
 model.compile(Optim.Adam(lr = 1e-5), loss = 'binary_crossentropy', metrics = ['accuracy'])
 return model
losses = []
models = []
crps_csv = []
s_time = time.time()
for k in range(2):
kfold = KFold(5, random_state = 42 + k, shuffle = True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards)) :
print("-----------")
 print("-----------")
 tr_x,tr_y = X[tr_inds],y[tr_inds]
val_x,val_y = X[val_inds],y[val_inds]
model,crps = get_model(tr_x,tr_y,val_x,val_y)
 models.append(model)
 print("the %d fold crps is %f"%(( k_fold+1),crps))
crps_csv.append(crps)
 print("mean crps is %f"%np.mean(crps_csv))
def predict(x_te):
model_num = len(models)
 for k,m in enumerate(models):
if k==0:
y_pred = m.predict(x_te,batch_size=1024)
 else:
y_pred+=m.predict(x_te,batch_size=1024)
 y_pred = y_pred / model_num
return y_pred 
<compute_test_metric>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
print("mean crps is %f"%np.mean(crps_csv)) 
<predict_on_test>
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
TRAIN_EPOCHS = 5
VALID_EPOCHS = 5
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
%%time
if TRAIN_OFFLINE==False:
env = nflrush.make_env()
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in iter_test:
basetable = create_features(test_df, deploy=True)
 basetable.drop(['GameId','PlayId'], axis=1, inplace=True)
 scaled_basetable = scaler.transform(basetable)
 y_pred = predict(scaled_basetable)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1 ).tolist() [0]
preds_df = pd.DataFrame(data=[y_pred], columns=sample_prediction_df.columns)
 env.predict(preds_df)
 env.write_submission_file()<set_options>
tokenizer = AutoTokenizer.from_pretrained(MODEL )
env = nflrush.make_env()<import_modules>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )
import keras
from keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping
from keras.optimizers import Adam
from keras.utils import plot_model
import keras.backend as K
import matplotlib.patches as patches<drop_column>
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=100000, random_state=0)
 ])
TARGET = 'Yards'
dropcols = [TARGET]
dropcols += ['HomeTeamAbbr', 'VisitorTeamAbbr', 'DisplayName', 'PlayerCollegeName',
"NflId", "GameId", 'PlayId', "Season", "NflIdRusher",
"JerseyNumber", 'PlayerBirthDate']
dropcols += ["GameWeather", 'FieldPosition', 'StadiumType', 'Location', 'Stadium',
'WindSpeed', "GameClock", 'Humidity', "Temperature",
'WindDirection', 'Turf']
dropcols += ['FieldPosition', 'OffensePersonnel', 'DefensePersonnel',
'PossessionTeam', 'Week', 'PlayDirection',
'Team', 'TeamName', 'TimeHandoff', 'TimeSnap', 'OffenseFormation']<load_from_csv>
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen = MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen = MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
train_df = pd.read_csv(".. /input/nfl-big-data-bowl-2020/train.csv", low_memory=False)
 print('Train size: ', train_df.shape )<feature_engineering>
train_dataset =(tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.cache() 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
train_df['TeamName'] = np.nan
train_df.loc[train_df['Team'] == 'home', 'TeamName'] = train_df.loc[train_df['Team'] == 'home', 'HomeTeamAbbr']
train_df.loc[train_df['Team'] == 'away', 'TeamName'] = train_df.loc[train_df['Team'] == 'away', 'VisitorTeamAbbr']<groupby>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
fillnan_dict_OffenseFormation = train_df.groupby('TeamName')['OffenseFormation'].apply(pd.Series.mode)
 fillnan_dict_StadiumType = train_df.groupby('HomeTeamAbbr')['StadiumType'].apply(pd.Series.mode)
 fillnan_dict_WindSpeed = train_df.groupby(['Week','GameId'])['WindSpeed'].apply(pd.Series.mode)
 fillnan_dict_WindDirection = train_df.groupby(['Week','GameId'])['WindDirection'].apply(pd.Series.mode)
 fillnan_dict_OffenseFormation.index = fillnan_dict_OffenseFormation.index.droplevel(1)
 fillnan_dict_OffenseFormation = fillnan_dict_OffenseFormation.to_dict()
fillnan_dict_StadiumType.index = fillnan_dict_StadiumType.index.droplevel(1)
 fillnan_dict_StadiumType = fillnan_dict_StadiumType.to_dict()
fillnan_dict_WindSpeed.index = fillnan_dict_WindSpeed.index.droplevel(1)
 fillnan_dict_WindSpeed = fillnan_dict_WindSpeed.to_dict()
fillnan_dict_WindDirection.index = fillnan_dict_WindDirection.index.droplevel(1)
 fillnan_dict_WindDirection = fillnan_dict_WindDirection.to_dict()
yardsleft_target_mean = None
oh = OneHotEncoder(sparse=False)
 le = LabelEncoder()
scaler = StandardScaler()
position_cols = list(train_df['Position'].unique())
 train_df['IsRusher'] = train_df.NflId == train_df.NflIdRusher
dropcols += ['IsRusher']<feature_engineering>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=5
)
def preprocess(train_df, test=False):
global dropcols,position_cols
train_df.loc[train_df.VisitorTeamAbbr == "ARI", 'VisitorTeamAbbr'] = "ARZ"
train_df.loc[train_df.HomeTeamAbbr == "ARI", 'HomeTeamAbbr'] = "ARZ"
train_df.loc[train_df.VisitorTeamAbbr == "BAL", 'VisitorTeamAbbr'] = "BLT"
train_df.loc[train_df.HomeTeamAbbr == "BAL", 'HomeTeamAbbr'] = "BLT"
train_df.loc[train_df.VisitorTeamAbbr == "CLE", 'VisitorTeamAbbr'] = "CLV"
train_df.loc[train_df.HomeTeamAbbr == "CLE", 'HomeTeamAbbr'] = "CLV"
train_df.loc[train_df.VisitorTeamAbbr == "HOU", 'VisitorTeamAbbr'] = "HST"
train_df.loc[train_df.HomeTeamAbbr == "HOU", 'HomeTeamAbbr'] = "HST"
WindMap={'SW':"SW", 'NNE':"NE", 'SE':"SE",'East':"E", np.nan:np.nan, 'NE':"NE", 'North':"N", 'S':"S", 'Northwest':"NW",
'SouthWest':"SW", 'ENE':"NE", 'ESE':"SE", 'SSW':"SW", 'NW':"NW", 'Northeast':"NE", 'From S':"S", 'W':"W",
'South':"S", 'West-Southwest':"SW", 'E':"E", '13':'13','N':"N", 'NNW':"NW",
'South Southeast':"SE", 'SSE':"SE", 'West':"W", 'WSW':"SW", 'From SW':"SW", 'WNW':"NW", 's':"S",
'NorthEast':"NE", 'from W':"W", 'W-NW':"NW", 'South Southwest':"SW", 'Southeast':"SE",
'From WSW':"SW", 'West Northwest':"NW", 'Calm':"Calm", 'From SSE':"SE", 'From W':"W",
'East North East':"NE", 'From ESE':"SE", 'EAST':"E", 'East Southeast':"SE",
'From SSW':"SW", '8':"8", 'North East':"NE", 'Southwest':"SW", 'North/Northwest':"NW",
'From NNE':"NE", '1':"NE", 'N-NE':"NE", 'W-SW':"SW", 'From NNW':"NW"}
Turf_map={"Grass":"Grass","grass":"Grass",'Twenty-Four/Seven Turf':"FieldTurf",
'DD GrassMaster':'DD GrassMaster','A-Turf Titan':'A-Turf Titan',
"FieldTurf 360":"FieldTurf","FieldTurf360":"FieldTurf",'UBU Sports Speed S5-M':'UBU Sports Speed S5-M',
"Field Turf":"FieldTurf","Natural":"Natural", "Natural Grass":"Natural",
"Natural grass":"Natural","Naturall Grass":"Natural",'SISGrass':'Grass',
'UBU Speed Series-S5-M':"UBU Speed Series-S5-M","Artifical":"Artifical","Artificial":"Artifical"}
Type_map={'Outdoor':'Outdoor','Outdoors':'Outdoor','Oudoor':'Outdoor'
,'Ourdoor':'Outdoor','Outside':'Outdoor','Outdor':'Outdoor'
,'Outddors':'Outdoor','Retractable Roof':'Retractable Roof',
'Retr.Roof-Closed':'Retractable Roof','Retr.Roof - Closed':'Retractable Roof',
'Retr.Roof Closed':'Retractable Roof', "Outdoor Retr Roof-Open":"Outdoor",
"Indoors":"Indoor","Indoor":"Indoor","Indoor, Roof Closed":"Indoor",
"Doom":"Doom","Domed, closed":"Doom","Domed, open":"Doom","Closed Dome":"Doom"
,"Bowl":"Bowl","Open":"Outdoor","Cloudy":"Outdoor","Domed, Open":"Doom"
,'Retr.Roof-Open':'Retractable Roof',"Retr.Roof - Open":"Retractable Roof",
"Indoor, Open Roof":"Indoor","Heinz Field" :"Heinz Field"}
if test:
train_df['TeamName'] = np.nan
train_df.loc[train_df['Team'] == 'home', 'TeamName'] = train_df.loc[train_df['Team'] == 'home', 'HomeTeamAbbr']
train_df.loc[train_df['Team'] == 'away', 'TeamName'] = train_df.loc[train_df['Team'] == 'away', 'VisitorTeamAbbr']
train_df.StadiumType = train_df.StadiumType.map(Type_map)
 train_df.Turf = train_df.Turf.map(Turf_map)
 train_df.WindDirection = train_df.WindDirection.map(WindMap)
 train_df['OffenseFormation'] = train_df.set_index('TeamName')['OffenseFormation'].fillna(fillnan_dict_OffenseFormation ).values
train_df['WindSpeed'] = train_df.set_index(['Week','GameId'])['WindSpeed'].fillna(fillnan_dict_WindSpeed ).values
train_df['WindDirection'] = train_df.set_index(['Week','GameId'])['WindDirection'].fillna(fillnan_dict_WindDirection ).values
def transform_gameclock(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans
train_df['GameClock'] = train_df['GameClock'].apply(transform_gameclock)
 transform_time = lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ")
 train_df['TimeHandoff'] = train_df['TimeHandoff'].apply(transform_time)
 train_df['TimeSnap'] = train_df['TimeSnap'].apply(transform_time)
 train_df['TimeDelta'] =(train_df['TimeHandoff'] - train_df['TimeSnap'] ).dt.seconds
transform_birthdaytime = lambda x: datetime.datetime.strptime(x, "%m/%d/%Y")
 train_df['PlayerBirthDate'] = train_df['PlayerBirthDate'].apply(transform_birthdaytime)
 seconds_in_year = 60*60*24*365.25
train_df['PlayerAge'] =(train_df['TimeHandoff'] - train_df['PlayerBirthDate'] ).dt.seconds / seconds_in_year
transform_playerheight = lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1])
 train_df['PlayerHeight'] = train_df['PlayerHeight'].apply(transform_playerheight)
 train_df['WindSpeed'] = train_df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 train_df['WindSpeed'] = train_df['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train_df['WindSpeed'] = train_df['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 def personnel_split(x, pattern):
s = re.findall(f"([1-9]){pattern}", x)
 if len(s)> 0:
return int(s[0])
 else:
return 0
train_df['OffensePersonnel_RB'] = train_df['OffensePersonnel'].apply(personnel_split, pattern='RB')
 train_df['OffensePersonnel_DB'] = train_df['OffensePersonnel'].apply(personnel_split, pattern='DB')
 train_df['OffensePersonnel_DL'] = train_df['OffensePersonnel'].apply(personnel_split, pattern='DL')
 train_df['OffensePersonnel_LB'] = train_df['OffensePersonnel'].apply(personnel_split, pattern='LB')
 train_df['OffensePersonnel_OL'] = train_df['OffensePersonnel'].apply(personnel_split, pattern='OL')
 train_df['OffensePersonnel_QB'] = train_df['OffensePersonnel'].apply(personnel_split, pattern='QB')
 train_df['OffensePersonnel_TE'] = train_df['OffensePersonnel'].apply(personnel_split, pattern='TE')
 train_df['OffensePersonnel_WR'] = train_df['OffensePersonnel'].apply(personnel_split, pattern='WR')
 train_df['DefensePersonnel_DB'] = train_df['DefensePersonnel'].apply(personnel_split, pattern='DB')
 train_df['DefensePersonnel_DL'] = train_df['DefensePersonnel'].apply(personnel_split, pattern='DL')
 train_df['DefensePersonnel_LB'] = train_df['DefensePersonnel'].apply(personnel_split, pattern='LB')
 train_df['DefensePersonnel_OL'] = train_df['DefensePersonnel'].apply(personnel_split, pattern='OL')
 train_df['Orientation'] = np.where(train_df['Season'] == 2017,(360+train_df['Orientation']+90 ).mod(360), train_df['Orientation'])
 train_df['Dir_rad'] = np.mod(90 - train_df.Dir, 360)* math.pi/180.0
train_df['YardLine_std'] = np.where(train_df.FieldPosition == train_df.PossessionTeam, train_df.YardLine, 100 - train_df.YardLine)
 train_df['X'] = np.where(train_df['PlayDirection'] == 'right', train_df['X'], 120-train_df['X'])
 train_df['Y'] = np.where(train_df['PlayDirection'] == 'right', train_df['Y'], 160/3 - train_df['Y'])
 train_df['Orientation'] = np.where(train_df['PlayDirection'] == 'right', train_df['Orientation'],(360 - train_df['Orientation'] ).mod(360))
train_df['Dir'] = np.where(train_df['PlayDirection'] == 'right', train_df['Dir'], np.mod(np.pi + train_df['Dir_rad'], 2*np.pi))
dropcols += ['YardLine']
tmp = np.where(train_df['FieldPosition'] == train_df['HomeTeamAbbr'], 100 - train_df['YardLine'], train_df['YardLine'])
 train_df['YardsLeft'] = np.where(train_df['PlayDirection'] == 'right', tmp, 100 - tmp)
 if not test:
train_df.drop(train_df.index[(train_df['YardsLeft'] < train_df['Yards'])|(( train_df['YardsLeft']-100)> train_df['Yards'])], inplace=True)
 del tmp
train_df['IsRusher'] =(train_df['NflId'] == train_df['NflIdRusher'])
 dropcols.append('IsRusher')
 if not test:
oh.fit(train_df[['OffenseFormation']])
 val = oh.transform(train_df[['OffenseFormation']])
 count = 0
for j in range(val.shape[1]):
train_df[f'x{count}'] = val[:, j]
count += 1
if not test:
tmp = train_df.groupby(by=['PlayId', 'Position'] ).size().unstack(level=1)
 tmp = tmp.fillna(0)
 tmp = tmp.astype(int)
 tmp.columns = [f"Position_{i}" for i in tmp.columns]
train_df = pd.merge(train_df, tmp, left_on='PlayId', right_index=True, how='left')
 else:
tmp = train_df.groupby(by=['PlayId', 'Position'] ).size().unstack(level=1)
 for c in position_cols:
if c not in tmp.columns:
tmp[c] = np.nan
tmp.columns = [f"Position_{i}" for i in tmp.columns]
tmp = tmp.fillna(0)
 tmp = tmp.astype(int)
 train_df = pd.merge(train_df, tmp, left_on='PlayId', right_index=True, how='left')
 dropcols += ['Position']
if not test:
tmp = pd.get_dummies(train_df['Down'])
 tmp.columns = [f"Down_{i}" for i in tmp.columns]
train_df = pd.concat([train_df, tmp], axis=1)
 else:
downs = [1, 2, 3, 4]
down = train_df['Down'].iloc[0]
downs.remove(down)
 train_df[f'Down_{down}'] = 1
for d in downs:
train_df[f'Down_{d}'] = 0
dropcols += ['Down']
if not test:
tmp = pd.get_dummies(train_df['Quarter'])
 tmp.columns = [f"Quarter_{i}" for i in tmp.columns]
train_df = pd.concat([train_df, tmp], axis=1)
 else:
downs = [1, 2, 3, 4, 5]
down = train_df['Quarter'].iloc[0]
downs.remove(down)
 train_df[f'Quarter_{down}'] = 1
for d in downs:
train_df[f'Quarter_{d}'] = 0
dropcols += ['Quarter']
train_df = feature_engineering(train_df, test)
 train_df = train_df.sort_values(by=['PlayId', 'Team', 'IsRusher', 'JerseyNumber'])
 return train_df<categorify>
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=5
)
%%time
train_df = preprocess(train_df )<count_missing_values>
model.save_weights('UnToxik_V0-1.h5' )
<count_unique_values><EOS> 
tokenizer.save_pretrained('.' )