Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<load_from_csv>
nltk.download('punkt')
 def seed_everything(seed=0):
np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
SEED = 40
seed_everything(SEED )
%%time
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32} )<prepare_x_and_y>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=True,
return_token_type_ids=True,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'],dtype=np.int32),np.array(enc_di['attention_mask'],dtype=np.int32),np.array(enc_di['token_type_ids'],dtype=np.int32 )
rows = 150000
segments = int(np.floor(train.shape[0] / rows))
X_tr = pd.DataFrame(index=range(segments), dtype=np.float64)
 y_tr = pd.DataFrame(index=range(segments), dtype=np.float64,
columns=['time_to_failure'])
 for segment in tqdm_notebook(range(segments)) :
seg = train.iloc[segment*rows:segment*rows+rows]
x_raw = seg['acoustic_data']
x = x_raw.values
y = seg['time_to_failure'].values[-1]
y_tr.loc[segment, 'time_to_failure'] = y
X_tr.loc[segment, 'ave'] = x.mean()
X_tr.loc[segment, 'std'] = x.std()
X_tr.loc[segment, 'max'] = x.max()
X_tr.loc[segment, 'min'] = x.min()
X_tr.loc[segment, 'q01'] = np.quantile(x,0.01)
 X_tr.loc[segment, 'q05'] = np.quantile(x,0.05)
 X_tr.loc[segment, 'q95'] = np.quantile(x,0.95)
 X_tr.loc[segment, 'q99'] = np.quantile(x,0.99)
 X_tr.loc[segment, 'abs_median'] = np.median(np.abs(x))
X_tr.loc[segment, 'abs_q95'] = np.quantile(np.abs(x),0.95)
 X_tr.loc[segment, 'abs_q99'] = np.quantile(np.abs(x),0.99)
 X_tr.loc[segment, 'F_test'], X_tr.loc[segment, 'p_test'] = stats.f_oneway(x[:30000],x[30000:60000],x[60000:90000],x[90000:120000],x[120000:])
 X_tr.loc[segment, 'av_change_abs'] = np.mean(np.diff(x))
X_tr.loc[segment, 'av_change_rate'] = np.mean(np.nonzero(( np.diff(x)/ x[:-1])) [0])
 X_tr.loc[segment, 'abs_max'] = np.abs(x ).max()
for windows in [10,100]:
x_roll_std = x_raw.rolling(windows ).std().dropna().values
x_roll_mean = x_raw.rolling(windows ).mean().dropna().values
X_tr.loc[segment, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X_tr.loc[segment, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X_tr.loc[segment, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X_tr.loc[segment, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X_tr.loc[segment, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.01)
 X_tr.loc[segment, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.05)
 X_tr.loc[segment, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.95)
 X_tr.loc[segment, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.99)
 X_tr.loc[segment, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X_tr.loc[segment, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X_tr.loc[segment, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X_tr.loc[segment, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X_tr.loc[segment, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X_tr.loc[segment, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X_tr.loc[segment, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
X_tr.loc[segment, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean,0.01)
 X_tr.loc[segment, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean,0.05)
 X_tr.loc[segment, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean,0.95)
 X_tr.loc[segment, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean,0.99)
 X_tr.loc[segment, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_tr.loc[segment, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X_tr.loc[segment, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max()<load_from_csv>
LANGS = {
'en': 'english',
'it': 'italian',
'fr': 'french',
'es': 'spanish',
'tr': 'turkish',
'ru': 'russian',
'pt': 'portuguese'
}
def get_sentences(text, lang='en'):
return sent_tokenize(text, LANGS.get(lang, 'english'))
def exclude_duplicate_sentences(text, lang='en'):
sentences = []
for sentence in get_sentences(text, lang):
sentence = sentence.strip()
if sentence not in sentences:
sentences.append(sentence)
 return ' '.join(sentences)
 def clean_text(text, lang='en'):
text = str(text)
 text = re.sub(r'[0-9"]', '', text)
 text = re.sub(r'
text = re.sub(r'@[\S]+\b', '', text)
 text = re.sub(r'https?\S+', '', text)
 text = re.sub(r'\s+', ' ', text)
 text = exclude_duplicate_sentences(text, lang)
 return text.strip()
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 X_test = pd.DataFrame(columns=X_tr.columns, dtype=np.float64, index=submission.index)
 plt.figure(figsize=(22, 16))
for i, seg_id in enumerate(tqdm_notebook(X_test.index)) :
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 x_raw = seg['acoustic_data']
x_roll = x_raw.rolling(windows ).std().dropna().values
x = x_raw.values
X_test.loc[seg_id, 'ave'] = x.mean()
X_test.loc[seg_id, 'std'] = x.std()
X_test.loc[seg_id, 'max'] = x.max()
X_test.loc[seg_id, 'min'] = x.min()
X_test.loc[seg_id, 'q01'] = np.quantile(x,0.01)
 X_test.loc[seg_id, 'q05'] = np.quantile(x,0.05)
 X_test.loc[seg_id, 'q95'] = np.quantile(x,0.95)
 X_test.loc[seg_id, 'q99'] = np.quantile(x,0.99)
 X_test.loc[seg_id, 'abs_median'] = np.median(np.abs(x))
X_test.loc[seg_id, 'abs_q95'] = np.quantile(np.abs(x),0.95)
 X_test.loc[seg_id, 'abs_q99'] = np.quantile(np.abs(x),0.99)
 X_test.loc[seg_id, 'F_test'], X_tr.loc[segment, 'p_test'] = stats.f_oneway(x[:30000],x[30000:60000],x[60000:90000],x[90000:120000],x[120000:])
 X_test.loc[seg_id, 'av_change_abs'] = np.mean(np.diff(x))
X_test.loc[seg_id, 'av_change_rate'] = np.mean(np.nonzero(( np.diff(x)/ x[:-1])) [0])
 X_test.loc[seg_id, 'abs_max'] = np.abs(x ).max()
for windows in [10,100]:
x_roll_std = x_raw.rolling(windows ).std().dropna().values
x_roll_mean = x_raw.rolling(windows ).mean().dropna().values
X_test.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X_test.loc[seg_id, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X_test.loc[seg_id, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X_test.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X_test.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.01)
 X_test.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.05)
 X_test.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.95)
 X_test.loc[seg_id, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.99)
 X_test.loc[seg_id, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X_test.loc[seg_id, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X_test.loc[seg_id, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X_test.loc[seg_id, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X_test.loc[seg_id, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X_test.loc[seg_id, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X_test.loc[seg_id, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
X_test.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean,0.01)
 X_test.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean,0.05)
 X_test.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean,0.95)
 X_test.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean,0.99)
 X_test.loc[seg_id, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_test.loc[seg_id, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X_test.loc[seg_id, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max()
if i < 12:
plt.subplot(3, 4, i + 1)
 plt.plot(seg['acoustic_data'])
 plt.title(seg_id )<choose_model_class>
def build_model(transformer, max_len=512):

 input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, _, hs = transformer(( input_ids,input_mask,segment_ids))
x = tf.stack([hs[-1],hs[-2],hs[-3]])
 sequence_output = tf.reduce_mean(x, axis = 0)
 pool_output = []
kernel_sizes = [3, 4, 5]
for kernel_size in kernel_sizes:
c = Conv1D(filters=64, kernel_size=kernel_size, padding='same',activation='relu',strides=1 )(sequence_output)
 p = GlobalMaxPooling1D()(c)
 pool_output.append(p)
 pool_output = concatenate([p for p in pool_output])
 out = pool_output
out = Dropout(0.3 )(out)
 out = Dense(64, activation='relu' )(out)
 out = Dropout(0.3 )(out)
 out = Dense(1, activation='sigmoid' )(out)
 model = Model(inputs=[input_ids,input_mask,segment_ids], outputs=out)
 model.compile(Adam(lr=0.2e-5), loss='binary_crossentropy', metrics=['accuracy',tf.keras.metrics.AUC() ])
 return model
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=11 )<split>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
def train_model(X=X_tr, X_test=X_test, y=y_tr, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, model=None):
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='mae',
verbose=1000, early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X_tr.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X_tr.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X_tr.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_tr.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'rcv':
model = RidgeCV(alphas=(0.01, 0.1, 1.0, 10.0, 100.0), scoring='neg_mean_absolute_error', cv=3)
 model.fit(X_train, y_train)
 print(model.alpha_)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = mean_absolute_error(y_valid, y_pred_valid)
 print(f'Fold {fold_n}.MAE: {score:.4f}.')
 print('')
 y_pred = model.predict(X_test ).reshape(-1,)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = mean_absolute_error(y_valid, y_pred_valid)
 print(f'Fold {fold_n}.MAE: {score:.4f}.')
 print('')
 y_pred = model.predict(X_test ).reshape(-1,)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=20000, eval_metric='MAE', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_absolute_error(y_valid, y_pred_valid))
prediction += y_pred
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}.'.format(mean_absolute_error(y, oof)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction, feature_importance
return oof, prediction
else:
return oof, prediction<init_hyperparams>
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 210
MODEL = 'jplu/tf-xlm-roberta-large'
PRETRAINED_MODEL = '/kaggle/input/jigsaw-mlm-finetuned-xlm-r-large'
params = {'num_leaves': 64,
'min_data_in_leaf': 50,
'objective': 'mae',
'max_depth': -1,
'learning_rate': 0.001,
"boosting": "gbdt",
"feature_fraction": 0.5,
"bagging_freq": 2,
"bagging_fraction": 0.5,
"bagging_seed": 0,
"metric": 'mae',
"verbosity": -1,
'reg_alpha': 1.0,
'reg_lambda': 1.0,
}
oof_lgb, prediction_lgb, feature_importance = train_model(params=params, model_type='lgb', plot_feature_importance=True )<init_hyperparams>
tokenizer = AutoTokenizer.from_pretrained(MODEL )
params = {'num_leaves': 128,
'min_data_in_leaf': 50,
'objective': 'mae',
'max_depth': -1,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.5,
"bagging_freq": 2,
"bagging_fraction": 0.7,
"bagging_seed": 0,
"metric": 'mae',
"verbosity": -1,
'reg_alpha': 2.0,
'reg_lambda': 1.5,
}
oof_lgb2, prediction_lgb2, feature_importance2 = train_model(params=params, model_type='lgb', plot_feature_importance=True )<init_hyperparams>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )
params = {'num_leaves': 32,
'min_data_in_leaf': 10,
'objective': 'mae',
'max_depth': -1,
'learning_rate': 0.001,
"boosting": "gbdt",
"feature_fraction": 0.8,
"bagging_freq": 2,
"bagging_fraction": 0.8,
"bagging_seed": 0,
"metric": 'mae',
"verbosity": -1,
'reg_alpha': 1.0,
'reg_lambda': 0.5,
}
oof_lgb3, prediction_lgb3, feature_importance3 = train_model(params=params, model_type='lgb', plot_feature_importance=True )<train_model>
train3 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-tr-cleaned.csv")
 train4 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-it-cleaned.csv")
 train5 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-pt-cleaned.csv")
 train6 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-ru-cleaned.csv")
 train7 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-fr-cleaned.csv")
 train8 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv" )
xgb_params = {'eta': 0.01, 'max_depth': 6, 'subsample': 0.8, 'colsample_bytree': 0.8,'colsample_bylevel': 0.8, 'colsample_bynode': 0.8,
'lambda': 0.1, 'alpha' : 0.1, 'objective': 'reg:linear', 'eval_metric': 'mae', 'silent': True, 'nthread': 4}
oof_xgb, prediction_xgb = train_model(params=xgb_params, model_type='xgb' )<train_model>
train1['toxic'].value_counts()
xgb_params = {'eta': 0.01, 'max_depth': 12, 'subsample': 0.5, 'colsample_bytree': 0.5,'colsample_bylevel': 0.5, 'colsample_bynode': 0.5,
'lambda': 0.5, 'alpha' : 0.5, 'objective': 'reg:linear', 'eval_metric': 'mae', 'silent': True, 'nthread': 4}
oof_xgb2, prediction_xgb2 = train_model(params=xgb_params, model_type='xgb' )<compute_test_metric>
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1' ).sample(n=60000, random_state=12312),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=60000, random_state=42323),
])
 train_tr = pd.concat([
train3[['comment_text', 'toxic']].query('toxic==1'),
train3[['comment_text', 'toxic']].query('toxic==0' ).sample(n=50000, random_state=23412),
])
 train_it = pd.concat([
train4[['comment_text', 'toxic']].query('toxic==1'),
train4[['comment_text', 'toxic']].query('toxic==0' ).sample(n=50000, random_state=23412),
])
 train_pt = pd.concat([
train5[['comment_text', 'toxic']].query('toxic==1'),
train5[['comment_text', 'toxic']].query('toxic==0' ).sample(n=50000, random_state=75293),
])
 train_ru = pd.concat([
train6[['comment_text', 'toxic']].query('toxic==1'),
train6[['comment_text', 'toxic']].query('toxic==0' ).sample(n=50000, random_state=17479),
])
 train_fr = pd.concat([
train7[['comment_text', 'toxic']].query('toxic==1'),
train7[['comment_text', 'toxic']].query('toxic==0' ).sample(n=50000, random_state=56321),
])
 train_es = pd.concat([
train8[['comment_text', 'toxic']].query('toxic==1'),
train8[['comment_text', 'toxic']].query('toxic==0' ).sample(n=50000, random_state=45874),
] )
print(mean_absolute_error(y_tr,(oof_lgb + oof_lgb2 + oof_lgb3 + oof_xgb + oof_xgb2)/5))<save_to_csv>
train['comment_text'] = train['comment_text'].apply(lambda x : clean_text(x))
valid['comment_text'] = valid['comment_text'].apply(lambda x : clean_text(x))
train_tr['comment_text'] = train_tr['comment_text'].apply(lambda x : clean_text(x,lang='tr'))
train_pt['comment_text'] = train_pt['comment_text'].apply(lambda x : clean_text(x,lang='pt'))
train_it['comment_text'] = train_it['comment_text'].apply(lambda x : clean_text(x,lang='it'))
train_ru['comment_text'] = train_ru['comment_text'].apply(lambda x : clean_text(x,lang='ru'))
train_fr['comment_text'] = train_fr['comment_text'].apply(lambda x : clean_text(x,lang='fr'))
train_es['comment_text'] = train_es['comment_text'].apply(lambda x : clean_text(x,lang='es'))
test['content'] = test['content'].apply(lambda x : clean_text(x))
submission['time_to_failure'] =(prediction_lgb + prediction_lgb2 + prediction_lgb3 + prediction_xgb + prediction_xgb2)/ 5
print(submission.head())
 submission.to_csv('submission_all.csv' )<import_modules>
%%time
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_train_tr = regular_encode(train_tr.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_train_pt = regular_encode(train_pt.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_train_it = regular_encode(train_it.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_train_ru = regular_encode(train_ru.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_train_fr = regular_encode(train_fr.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_train_es = regular_encode(train_es.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_train_tr = train_tr.toxic.values
y_train_pt = train_pt.toxic.values
y_train_it = train_it.toxic.values
y_train_ru = train_ru.toxic.values
y_train_fr = train_fr.toxic.values
y_train_es = train_es.toxic.values
y_valid = valid.toxic.values
import matplotlib.pyplot as plt
from tqdm import tqdm
from sklearn.preprocessing import StandardScaler
from sklearn.svm import NuSVR
from sklearn.metrics import mean_absolute_error<load_from_csv>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.shuffle(len(train)) 
.batch(BATCH_SIZE)
.repeat() 
.prefetch(AUTO)
)
 train_tr_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train_tr, y_train_tr)) 
.shuffle(len(train_tr)) 
.batch(BATCH_SIZE)
.repeat() 
.prefetch(AUTO)
)
 train_it_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train_it, y_train_it)) 
.shuffle(len(train_it)) 
.batch(BATCH_SIZE)
.repeat() 
.prefetch(AUTO)
)
 train_pt_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train_pt, y_train_pt)) 
.shuffle(len(train_pt)) 
.batch(BATCH_SIZE)
.repeat() 
.prefetch(AUTO)
)
 train_ru_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train_ru, y_train_ru)) 
.shuffle(len(train_ru)) 
.batch(BATCH_SIZE)
.repeat() 
.prefetch(AUTO)
)
 train_fr_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train_fr, y_train_fr)) 
.shuffle(len(train_fr)) 
.batch(BATCH_SIZE)
.repeat() 
.prefetch(AUTO)
)
 train_es_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train_es, y_train_es)) 
.shuffle(len(train_es)) 
.batch(BATCH_SIZE)
.repeat() 
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64} )<set_options>
%%time
with strategy.scope() :
config = AutoConfig.from_pretrained(PRETRAINED_MODEL)
 config.output_hidden_states = True
transformer_layer = TFAutoModel.from_pretrained(PRETRAINED_MODEL,config=config)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
pd.options.display.precision = 15<prepare_x_and_y>
n_steps = train_tr.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_tr_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS+1,
shuffle=False,
)
rows = 150_000
segments = int(np.floor(train.shape[0] / rows))
X_train = pd.DataFrame(index=range(segments), dtype=np.float64,
columns=['ave', 'std', 'max', 'min'])
 y_train = pd.DataFrame(index=range(segments), dtype=np.float64,
columns=['time_to_failure'])
 for segment in tqdm(range(segments)) :
seg = train.iloc[segment*rows:segment*rows+rows]
x = seg['acoustic_data'].values
y = seg['time_to_failure'].values[-1]
y_train.loc[segment, 'time_to_failure'] = y
X_train.loc[segment, 'ave'] = x.mean()
X_train.loc[segment, 'std'] = x.std()
X_train.loc[segment, 'max'] = x.max()
X_train.loc[segment, 'min'] = x.min()<normalization>
n_steps = train_it.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_it_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
shuffle=False,
)
scaler = StandardScaler()
scaler.fit(X_train)
 X_train_scaled = scaler.transform(X_train )<train_on_grid>
n_steps = train_pt.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_pt_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
shuffle=False,
)
svm = NuSVR()
svm.fit(X_train_scaled, y_train.values.flatten())
 y_pred = svm.predict(X_train_scaled )<compute_test_metric>
n_steps = train_ru.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_ru_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
shuffle=False,
)
score = mean_absolute_error(y_train.values.flatten() , y_pred)
 print(f'Score: {score:0.3f}' )<load_from_csv>
n_steps = train_fr.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_fr_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
shuffle=False,
)
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id' )<create_dataframe>
n_steps = train_es.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_es_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
shuffle=False,
)
X_test = pd.DataFrame(columns=X_train.columns, dtype=np.float64, index=submission.index)
<feature_engineering>
%%time
n_steps = valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=5,
verbose=2,
shuffle=False,
)
<save_to_csv><EOS> 
sub['toxic'] = model.predict(x_test, verbose=1)
 sub.to_csv('submission.csv', index=False)
 sub.head(10 )