Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<split>
warnings.filterwarnings("ignore")
 tqdm.pandas()
CountVectorizer,\
HashingVectorizer
stopword=set(STOPWORDS)
 lem = WordNetLemmatizer()
tokenizer=TweetTokenizer()
np.random.seed(0)
env = twosigmanews.make_env() 
(market_train_df, news_train_df)= env.get_training_data()
market_train, news_train = market_train_df.copy() , news_train_df.copy()<categorify>
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print(strategy.num_replicas_in_sync)
 BATCH_SIZE = 16 * strategy.num_replicas_in_sync
def mis_impute(data):
for i in data.columns:
if data[i].dtype == "object":
data[i] = data[i].fillna("other")
 elif(data[i].dtype == "int64" or data[i].dtype == "float64"):
data[i] = data[i].fillna(data[i].mean())
 else:
pass
return data
market_train_df = mis_impute(market_train_df)
 def data_prep(market_train):
market_train.time = market_train.time.dt.date
lbl = {k: v for v, k in enumerate(market_train['assetCode'].unique())}
market_train['assetCodeT'] = market_train['assetCode'].map(lbl)
 market_train = market_train.dropna(axis=0)
 return market_train
market_train = data_prep(market_train_df)
 print(market_train.shape )<feature_engineering>
MODEL = 'jplu/tf-xlm-roberta-base'
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
 text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
tokenized_text = tokenizer.tokenize(text )
market_train = market_train.loc[market_train['time']>=date(2009, 1, 1)]
up = market_train.returnsOpenNextMktres10 >= 0
fcol = [c for c in market_train if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', 'universe','sourceTimestamp']]
X = market_train[fcol].values
up = up.values
r = market_train.returnsOpenNextMktres10.values
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng)
 assert X.shape[0] == up.shape[0] == r.shape[0]
X_train, X_test, up_train, up_test, r_train, r_test = model_selection.train_test_split(X, up, r, test_size=0.25, random_state=99)
 train_data = lgb.Dataset(X_train, label=up_train.astype(int))
test_data = lgb.Dataset(X_test, label=up_test.astype(int))<train_model>
masked_index = 8
tokenized_text[masked_index] = '[MASK]'
assert tokenized_text == ['[CLS]', 'who', 'was', 'jim', 'henson', '?', '[SEP]', 'jim', '[MASK]', 'was', 'a', 'puppet', '
indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
 segments_ids = [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1]
tokens_tensor = torch.tensor([indexed_tokens])
 segments_tensors = torch.tensor([segments_ids] )
<init_hyperparams>
model = BertForMaskedLM.from_pretrained('bert-base-uncased')
 model.eval()
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print(device)
 tokens_tensor = tokens_tensor.to(device)
 segments_tensors = segments_tensors.to(device)
 model.to(device)
 with torch.no_grad() :
outputs = model(tokens_tensor, token_type_ids=segments_tensors)
 predictions = outputs[0]
predicted_index = torch.argmax(predictions[0, masked_index] ).item()
predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0]
assert predicted_token == 'henson'
print('Predicted token is:',predicted_token )
x_1 = [0.19000424246380565, 2452, 212, 328, 202]
x_2 = [0.19016805202090095, 2583, 213, 312, 220]
params_1 = {
'task': 'train',
'boosting_type': 'dart',
'objective': 'binary',
'learning_rate': x_1[0],
'num_leaves': x_1[1],
'min_data_in_leaf': x_1[2],
'num_iteration': x_1[3],
'max_bin': x_1[4],
'verbose': 1
}
params_2 = {
'task': 'train',
'boosting_type': 'dart',
'objective': 'binary',
'learning_rate': x_2[0],
'num_leaves': x_2[1],
'min_data_in_leaf': x_2[2],
'num_iteration': x_2[3],
'max_bin': x_2[4],
'verbose': 1
}
gbm_1 = lgb.train(params_1,
train_data,
num_boost_round=100,
valid_sets=test_data,
early_stopping_rounds=5)
 gbm_2 = lgb.train(params_2,
train_data,
num_boost_round=100,
valid_sets=test_data,
early_stopping_rounds=5)
 days = env.get_prediction_days()
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
if(n_days%50==0):
print(n_days,end=' ')
 t = time.time()
market_obs_df = data_prep(market_obs_df)
 market_obs_df = market_obs_df[market_obs_df.assetCode.isin(predictions_template_df.assetCode)]
X_live = market_obs_df[fcol].values
X_live = 1 -(( maxs - X_live)/ rng)
 prep_time += time.time() - t
t = time.time()
lp =(gbm_1.predict(X_live)+ gbm_2.predict(X_live)) /2
prediction_time += time.time() -t
t = time.time()
confidence = lp
confidence =(confidence-confidence.min())/(confidence.max() -confidence.min())
 confidence = confidence * 2 - 1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':confidence})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
env.write_submission_file()
sub = pd.read_csv("submission.csv" )<import_modules>
import pandas as pd
from kaggle.competitions import twosigmanews
from datetime import datetime, date
import numpy as np
from sklearn import model_selection<split>
import pandas as pd
env = twosigmanews.make_env() 
(market_train_df, news_train_df)= env.get_training_data()
market_train, news_train = market_train_df.copy() , news_train_df.copy()<data_type_conversions>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 train3 = pd.read_csv('.. /input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv')
 train4 = pd.read_csv('.. /input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv')
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 toxic = len(train2[['comment_text', 'toxic']].query('toxic==1'))
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train3[['comment_text', 'toxic']].query('toxic==0'),
train3[['comment_text', 'toxic']].query('toxic==1'),
train4[['comment_text', 'toxic']].query('toxic==0'),
train4[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=(toxic+(toxic//3)) , random_state=101)
 ])
 test_data = test
train_data = train
maxlen = 192
def fill_in(data):
for i in data.columns:
if data[i].dtype == "object":
data[i] = data[i].fillna("")
 elif(data[i].dtype == "int64" or data[i].dtype == "float64"):
data[i] = data[i].fillna(data[i].mean())
 else:
pass
return data
market_train_df = fill_in(market_train_df )<categorify>
df_valid_en = pd.read_csv('.. /input/val-en-df/validation_en.csv')
 df_valid_en = df_valid_en.drop(columns=['id', 'comment_text','lang'])
 df_valid_en = df_valid_en.rename(columns={"comment_text_en": "comment_text"})
 columns_titles = ["comment_text","toxic"]
df_valid_en=df_valid_en.reindex(columns=columns_titles)

print(df_valid_en.head(3))
train_data = pd.concat([train_data , df_valid_en], axis=0 ).reset_index(drop=True)
 train_data = train_data.sample(frac=1 ).reset_index(drop=True)
 print(len(train2[['comment_text', 'toxic']].query('toxic==1')))
 print(len(train_data))
train_data.head()
def data_other(market_train_df):
market_train_df.time = market_train_df.time.dt.date
lbl = {k: v for v, k in enumerate(market_train_df['assetCode'].unique())}
market_train_df['assetCodeT'] = market_train_df['assetCode'].map(lbl)
 market_train_df = market_train_df.dropna(axis=0)
 return market_train_df
market_train_df = data_other(market_train_df )<filter>
val = valid
train = train_data
def clean(text):
text = text.fillna("fillna" ).str.lower()
text = text.map(lambda x: re.sub('\
',' ',str(x)))
 text = text.map(lambda x: re.sub("\[\[User.*",'',str(x)))
 text = text.map(lambda x: re.sub("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}",'',str(x)))
 text = text.map(lambda x: re.sub("\(http://.*?\s\(http://.*\)",'',str(x)))
 return text
val["comment_text"] = clean(val["comment_text"])
 test_data["content"] = clean(test_data["content"])
 train["comment_text"] = clean(train["comment_text"] )
market_train_df = market_train_df.loc[market_train_df['time']>=date(2009, 1, 1)]<filter>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '\xa0', '\t',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '\u3000', '\u202f',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '«',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"couldnt" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"doesnt" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"havent" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"shouldnt" : "should not",
"that's" : "that is",
"thats" : "that is",
"there's" : "there is",
"theres" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"theyre": "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def clean_text(x):
x = str(x ).replace("
","")
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
green = market_train_df.returnsOpenNextMktres10 > 0
green = green.values<define_variables>
tokenizer = TreebankWordTokenizer()
def handle_contractions(x):
x = tokenizer.tokenize(x)
 return x
def fix_quote(x):
x = [x_[1:] if x_.startswith("'")else x_ for x_ in x]
x = ' '.join(x)
 return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
def replace_typical_misspell(text):
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def clean_data(df, columns: list):
for col in columns:
df[col] = df[col].apply(lambda x: clean_text(x.lower()))
df[col] = df[col].apply(lambda x: replace_typical_misspell(x))
df[col] = df[col].apply(lambda x: handle_contractions(x))
df[col] = df[col].apply(lambda x: fix_quote(x))
return df
fcol = [c for c in market_train_df if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', 'universe','sourceTimestamp']]<define_variables>
%%time
input_columns = [
'comment_text'
]

 train = clean_data(train, input_columns)
 val = clean_data(val, input_columns)
X = market_train_df[fcol].values
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng )<define_variables>
%%time
input_columns = [
'content'
]
test_data = clean_data(test_data, input_columns)
 del tokenizer
all_10day = market_train_df.returnsOpenNextMktres10.values<split>
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))
X_train, X_test, green_train, green_test, all_train, all_test = model_selection.train_test_split(
 X, green, all_10day, test_size=0.20, random_state=59 )<data_type_conversions>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )
train_data = lgb.Dataset(X_train, label=green_train.astype(int))
test_data = lgb.Dataset(X_test, label=green_test.astype(int))<prepare_x_and_y>
MODEL = 'jplu/tf-xlm-roberta-large'
tokenizer = AutoTokenizer.from_pretrained(MODEL)
 save_path = '/kaggle/working/xlmr_large/'
if not os.path.exists(save_path):
os.makedirs(save_path)
 tokenizer.save_pretrained(save_path)
x_1 = [0.19000424246380565, 2452, 212, 328, 202]
x_2 = [0.19016805202090095, 2583, 213, 312, 220]<init_hyperparams>
%%time
x_train = regular_encode(train.comment_text.astype(str),
tokenizer, maxlen=maxlen)
 x_valid = regular_encode(val.comment_text.astype(str ).values,
tokenizer, maxlen=maxlen)
 x_test = regular_encode(test_data.content.astype(str ).values,
tokenizer, maxlen=maxlen)
 y_valid = val.toxic.values
y_train = train.toxic.values
params_1 = {
'task': 'train',
'boosting_type': 'dart',
'objective': 'binary',
'learning_rate': x_1[0],
'num_leaves': x_1[1],
'min_data_in_leaf': x_1[2],
'num_iteration': x_1[3],
'max_bin': x_1[4],
'verbose': 1
}
params_2 = {
'task': 'train',
'boosting_type': 'dart',
'objective': 'binary',
'learning_rate': x_2[0],
'num_leaves': x_2[1],
'min_data_in_leaf': x_2[2],
'num_iteration': x_2[3],
'max_bin': x_2[4],
'verbose': 1
}
gbm_1 = lgb.train(params_1,
train_data,
num_boost_round=100,
valid_sets=test_data,
early_stopping_rounds=5)
 gbm_2 = lgb.train(params_2,
train_data,
num_boost_round=100,
valid_sets=test_data,
early_stopping_rounds=5 )<define_variables>
%%time
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
days = env.get_prediction_days()
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0<predict_on_test>
def focal_loss(gamma=2., alpha=.2):
def focal_loss_fixed(y_true, y_pred):
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
return -K.mean(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) - K.mean(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return focal_loss_fixed
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
if(n_days%50==0):
print(n_days,end=' ')
 market_obs_df = fill_in(market_obs_df)
 market_obs_df = data_other(market_obs_df)
 market_obs_df = market_obs_df[market_obs_df.assetCode.isin(predictions_template_df.assetCode)]
X_live = market_obs_df[fcol].values
X_live = 1 -(( maxs - X_live)/ rng)
 lp =(gbm_1.predict(X_live)+ gbm_2.predict(X_live)) /2
confidence = lp
confidence =(confidence-confidence.min())/(confidence.max() -confidence.min())
 confidence = confidence * 2 - 1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':confidence})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df )<save_to_csv>
def build_model(transformer, loss='binary_crossentropy', max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
x = tf.keras.layers.Dropout(0.3 )(cls_token)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=3e-5), loss=loss, metrics=[tf.keras.metrics.AUC() ])
 return model
env.write_submission_file()<import_modules>
%%time
with strategy.scope() :
transformer_layer = transformers.TFXLMRobertaModel.from_pretrained(MODEL)
 model = build_model(transformer_layer,loss='binary_crossentropy', max_len=maxlen)
 model.summary()
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
from sklearn.metrics import accuracy_score
from kaggle.competitions import twosigmanews<split>
def callback() :
cb = []
reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss',
factor=0.3, patience=2,
verbose=1, mode='auto',
epsilon=0.0001, cooldown=1, min_lr=0.000001)
 cb.append(reduceLROnPlat)
 log = CSVLogger('log.csv')
 cb.append(log)
 RocAuc = RocAucEvaluation(validation_data=(x_valid, y_valid), interval=1)
 cb.append(RocAuc)
 return cb
env = twosigmanews.make_env() 
(market_train, _)= env.get_training_data()<define_variables>
def build_lrfn(lr_start=0.000001, lr_max=0.000002,
lr_min=0.0000001, lr_rampup_epochs=7,
lr_sustain_epochs=0, lr_exp_decay=.87):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
cat_cols = ['assetCode']
num_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10'] 
<split>
model_path = 'jigsawMultilingual.hdf5'
model_path1 = '/kaggle/working/jigsawMultilingual.hdf5'
train_indices, val_indices = train_test_split(market_train.index.values,test_size=0.25, random_state=23)
<categorify>
checkpoint = ModelCheckpoint(model_path, monitor='val_accuracy', mode='max', save_best_only=True)
 es = EarlyStopping(monitor='val_accuracy', mode='max', patience=2,
restore_best_weights=True, verbose=1)
 lr_callback = LearningRateScheduler(lrfn, verbose=1)
 callback_list = [checkpoint, lr_callback]
def encode(encoder, x):
len_encoder = len(encoder)
 try:
id = encoder[x]
except KeyError:
id = len_encoder
return id
encoders = [{} for cat in cat_cols]
for i, cat in enumerate(cat_cols):
print('encoding %s...' % cat, end=' ')
 encoders[i] = {l: id for id, l in enumerate(market_train.loc[train_indices, cat].astype(str ).unique())}
market_train[cat] = market_train[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
print('Done')
 embed_sizes = [len(encoder)+ 1 for encoder in encoders] 
<normalization>
%%time
N_STEPS = x_train.shape[0] // BATCH_SIZE
EPOCHS = 2
train_history = model.fit(
 train_dataset,
steps_per_epoch=N_STEPS,
validation_data=valid_dataset,
callbacks=callback_list,
epochs=EPOCHS
)
market_train[num_cols] = market_train[num_cols].fillna(0)
 print('scaling numerical columns')
 scaler = StandardScaler()
scaler = StandardScaler()
market_train[num_cols] = scaler.fit_transform(market_train[num_cols])
<choose_model_class>
if os.path.exists(model_path1):
model.load_weights(model_path1 )
categorical_inputs = []
for cat in cat_cols:
categorical_inputs.append(Input(shape=[1], name=cat))
categorical_embeddings = []
for i, cat in enumerate(cat_cols):
categorical_embeddings.append(Embedding(embed_sizes[i], 10 )(categorical_inputs[i]))
categorical_logits = Flatten()(categorical_embeddings[0])
 categorical_logits = Dense(32,activation='relu' )(categorical_logits)
 numerical_inputs = Input(shape=(11,), name='num')
 numerical_logits = numerical_inputs
numerical_logits = BatchNormalization()(numerical_logits)
 numerical_logits = Dense(128,activation='relu' )(numerical_logits)
 numerical_logits = Dense(64,activation='relu' )(numerical_logits)
 logits = Concatenate()([numerical_logits,categorical_logits])
 logits = Dense(64,activation='relu' )(logits)
 out = Dense(1, activation='sigmoid' )(logits)
 model = Model(inputs = categorical_inputs + [numerical_inputs], outputs=out)
 model.compile(optimizer='adam',loss=binary_crossentropy )<prepare_x_and_y>
%%time
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
callbacks=callback_list,
epochs= EPOCHS
)
def get_input(market_train, indices):
X_num = market_train.loc[indices, num_cols].values
X = {'num':X_num}
for cat in cat_cols:
X[cat] = market_train.loc[indices, cat_cols].values
y =(market_train.loc[indices,'returnsOpenNextMktres10'] >= 0 ).values
r = market_train.loc[indices,'returnsOpenNextMktres10'].values
u = market_train.loc[indices, 'universe']
d = market_train.loc[indices, 'time'].dt.date
return X,y,r,u,d
X_train,y_train,r_train,u_train,d_train = get_input(market_train, train_indices)
 X_valid,y_valid,r_valid,u_valid,d_valid = get_input(market_train, val_indices )<train_model>
if os.path.exists(model_path1):
model.load_weights(model_path1 )
check_point = ModelCheckpoint('model.hdf5',verbose=True, save_best_only=True)
 early_stop = EarlyStopping(patience=5,verbose=True)
 model.fit(X_train,y_train.astype(int),
validation_data=(X_valid,y_valid.astype(int)) ,
epochs=2,
verbose=True,
callbacks=[early_stop,check_point] )<compute_test_metric>
log_dir = "/kaggle/working/log.csv"
if os.path.exists(log_dir):
os.remove(log_dir )
<define_variables><EOS> 
sub = pd.read_csv('.. /input/jigsaw-multilingual-toxic-comment-classification/' + 'sample_submission.csv')
 sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )