Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<compute_train_metric>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
TIME_FUNCTIONS = True
two_sigma_model = model2SigmaStockPrizes(market_train, news_train, verbose=True )<set_options>
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
tokenizer = AutoTokenizer.from_pretrained(MODEL)
 def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'])
 def build_model_PT(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC() ])
 return model
def build_model_mix(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_lang_tags = Input(shape=(4,), dtype=tf.float32, name="input_lang_tags")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
x = Concatenate()([cls_token, input_lang_tags])
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[input_word_ids, input_lang_tags], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC() ])
 return model
def lang_embed(lang, tran):
lang_codes = {'en':'000', 'es':'100', 'fr':'010',
'it':'001', 'pt':'110', 'ru':'101',
'tr':'011'}
tran_codes = {'orig':'0', 'tran':'1'}
vec = lang_codes[lang]+tran_codes[tran]
vec = [int(v)for v in vec]
return vec
def text_process(text):
ws = text.split(' ')
 if(len(ws)>160):
text = ' '.join(ws[:160])+ ' ' + ' '.join(ws[-32:])
 return text
del news_train
del market_train
gc.collect()
show_mem_usage()<filter>
test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv')
 test['content'] = test['content'].apply(lambda x: text_process(x))
x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 lang_tag_test = np.array([lang_embed(row['lang'], 'orig')for _, row in test.iterrows() ])
@timeit
def select_dates(self, first_date):
self.market = self.market.loc[self.market['time'] >= first_date]
self.news = self.news.loc[self.news['time'] >= first_date]
if self.verbose:
print("data before '{}' has been removed".format(first_date))
two_sigma_model.select_dates = select_dates.__get__(two_sigma_model)
 two_sigma_model.select_dates(datetime.datetime(2009, 1, 1, 0, 0, 0, 0, pytz.UTC))<set_options>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model_PT(transformer_layer, max_len=MAX_LEN)
gc.collect()
show_mem_usage()<categorify>
model.load_weights('/kaggle/input/mymodels/mg2mp4.h5')
 test['p1'] = model.predict(x_test, verbose=1)
@timeit
def column_combinations(self):
self.market['price_diff'] = self.market['close'] - self.market['open']
@timeit
def asset_codes_encoding(self):
self.news['assetCodesLen'] = self.news['assetCodes'].map(lambda x: len(eval(x)))
 self.news['assetCodes'] = self.news['assetCodes'].str.findall(f"'([\w\./]+)'" ).apply(lambda x: x[0])
 @timeit
def news_count(self):
t = self.news.groupby(['time', 'assetName'] ).size().reset_index(name='news_count')
 self.news = pd.merge(self.news, t, on=['time', 'assetName'])
 two_sigma_model.column_combinations = column_combinations.__get__(two_sigma_model)
 two_sigma_model.asset_codes_encoding = asset_codes_encoding.__get__(two_sigma_model)
 two_sigma_model.news_count = news_count.__get__(two_sigma_model)
 two_sigma_model.column_combinations()
two_sigma_model.asset_codes_encoding()
two_sigma_model.news_count()<feature_engineering>
model.load_weights('/kaggle/input/mymodels/en2mp1.h5')
 test['p2'] = model.predict(x_test, verbose=1)
@timeit
def aggregate_numericals(self):
numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
self.news_numerical = self.news.select_dtypes(include=numerics ).copy()
self.news_numerical_columns = self.news_numerical.columns
self.news_numerical.loc[:, 'assetCodes'] = self.news['assetCodes']
self.news_numerical.loc[:, 'time'] = self.news['time']
agg_func = {
'takeSequence': ['sum', 'mean', 'max', 'min', 'std'],
'bodySize': ['sum', 'mean', 'max', 'min', 'std'],
'marketCommentary': ['sum', 'mean'],
'sentenceCount': ['sum', 'mean', 'max', 'min', 'std'],
'wordCount': ['sum', 'mean', 'max', 'min', 'std'],
'relevance': ['sum', 'mean', 'max', 'min', 'std'],
'firstMentionSentence': ['sum', 'mean', 'max', 'min', 'std'],
'sentimentNegative': ['sum', 'mean', 'max', 'min', 'std'],
'sentimentNeutral': ['sum', 'mean', 'max', 'min', 'std'],
'sentimentPositive': ['sum', 'mean', 'max', 'min', 'std'],
'sentimentWordCount': ['sum', 'mean', 'max', 'min', 'std'],
'noveltyCount12H': ['sum', 'mean', 'max', 'min', 'std'],
'noveltyCount24H': ['sum', 'mean', 'max', 'min', 'std'],
'noveltyCount3D': ['sum', 'mean', 'max', 'min', 'std'],
'noveltyCount5D': ['sum', 'mean', 'max', 'min', 'std'],
'noveltyCount7D': ['sum', 'mean', 'max', 'min', 'std'],
'volumeCounts12H': ['sum', 'mean', 'max', 'min', 'std'],
'volumeCounts24H': ['sum', 'mean', 'max', 'min', 'std'],
'volumeCounts3D': ['sum', 'mean', 'max', 'min', 'std'],
'volumeCounts5D': ['sum', 'mean', 'max', 'min', 'std'],
'volumeCounts7D': ['sum', 'mean', 'max', 'min', 'std'],
'news_count': ['max']
}
self.news_numerical = self.news_numerical.groupby(['assetCodes', 'time'] ).agg(agg_func)
 self.news_numerical.columns = ['_'.join(col ).strip()
for col in self.news_numerical.columns.values]
self.news_numerical.reset_index(inplace=True)
 two_sigma_model.aggregate_numericals = aggregate_numericals.__get__(two_sigma_model)
<groupby>
model.load_weights('/kaggle/input/mymodels/en2mp4.h5')
 test['p3'] = model.predict(x_test, verbose=1)
@timeit
def aggregate_categoricals(self):
categ_columns = [col for col in self.news.columns
if col not in self.news_numerical_columns]
if self.verbose:
print("news's categorical columns:
", categ_columns)
 temp = self.news[self.news['news_count'] > 1][categ_columns].copy()
multiple_articles = temp.groupby(['assetCodes', 'time'] ).tail(1)
 single_articles = self.news[self.news['news_count'] == 1][categ_columns]
self.news_categ = pd.concat([multiple_articles, single_articles])
 two_sigma_model.aggregate_categoricals = aggregate_categoricals.__get__(two_sigma_model)
<merge>
del model
K.clear_session()
gc.collect()
@timeit
def merge_news(self):
self.news = pd.merge(self.news_numerical,
self.news_categ,
on = ['assetCodes', 'time'],
how='left')
 self.news_categ = 0
self.news_numerical = 0
@timeit
def merge_market_news(self, keep_news=False):
if keep_news:
self.merged_df = pd.merge(self.market,
self.news,
left_on=['time', 'assetCode'],
right_on=['time', 'assetCodes'],
how='left')
 else:
self.merged_df = self.market
self.news = 0
self.market = 0
self.merged_df = reduce_mem_usage(self.merged_df, self.verbose)
 if self.verbose:
print("merged_df's shape is: {}".format(self.merged_df.shape))
two_sigma_model.merge_news = merge_news.__get__(two_sigma_model)
 two_sigma_model.merge_market_news = merge_market_news.__get__(two_sigma_model)
 two_sigma_model.merge_market_news()<set_options>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model_mix(transformer_layer, max_len=MAX_LEN)
gc.collect()
show_mem_usage()<categorify>
model.load_weights('/kaggle/input/mymodels/mixmoriggen.h5')
 test['p4'] = model.predict([x_test, lang_tag_test], verbose=1)
@timeit
def set_labels(self, labeled_columns, define_index=False):
if define_index:
self.indexer = {}
for col in labeled_columns:
_, self.indexer[col] = pd.factorize(self.merged_df[col])
 self.categorical_columns = labeled_columns
if self.verbose:
print("categorical variables: {}".format(labeled_columns))
for col in labeled_columns:
self.merged_df[col] = self.indexer[col].get_indexer(self.merged_df[col])
 two_sigma_model.set_labels = set_labels.__get__(two_sigma_model)
 two_sigma_model.set_labels([], define_index=True )<set_options>
model.load_weights('/kaggle/input/mymodels/mixmorigp3.h5')
 test['p5'] = model.predict([x_test, lang_tag_test], verbose=1)
gc.collect()
show_mem_usage()<define_search_space>
mybest = pd.read_csv('/kaggle/input/mybest/sub9523.csv')
 X = np.array([test.p1.values, test.p2.values, test.p3.values, test.p4.values, test.p5.values] ).T
y = mybest.toxic.values
X_T, X_val, y_T, y_val = train_test_split(X, y, test_size=0.2)
 model = LinearRegression()
model.fit(X, y)
 prds = model.predict(X_val)
 score1 = roc_auc_score(y_val.round().astype(int), prds)
 score2 = roc_auc_score(prds.round().astype(int), y_val)
 print('Validation:', score1, score2)
 prds = model.predict(X)
 score1 = roc_auc_score(y.round().astype(int), prds)
 score2 = roc_auc_score(prds.round().astype(int), y)
 print('Again my best:', score1, score2)
 test['prd'] = prds
def run_parallel(grouped, method, verbose):
with parallel.Parallel(n_jobs=-1, verbose=verbose)as par:
segments = par(delayed(method )(df_seg)for df_seg in grouped)
 return segments
def account_for_past(df_code, n_lag=[7, 14, 21], shift_size=1):
features = [c for c in df_code.columns if c not in ['assetCode']]
lag_columns = []
for col in features:
for window in n_lag:
if LAST_DATE_ONLY:
tmp = df_code[df_code.index.max() - 2*timedelta(window):df_code.index.max() ]
rolled = tmp[col].shift(shift_size, freq='D' ).rolling(window=window)
 else:
rolled = df_code[col].shift(shift_size, freq='D' ).rolling(window=window)
 col_name = '{}_median_past{}'.format(col, window)
 lag_columns.append(col_name)
 df_code[col_name] = rolled.median()
col_name = '{}_max_past{}'.format(col, window)
 lag_columns.append(col_name)
 df_code[col_name] = rolled.max()
col_name = '{}_min_past{}'.format(col, window)
 lag_columns.append(col_name)
 df_code[col_name] = rolled.min()
return lag_columns, df_code.drop(features, axis=1)
 def create_returns(df_code, n_days=[3, 5, 7, 12]):
features = [c for c in df_code.columns if c not in ['assetCode']]
lag_columns = []
for col in features:
for days in n_days:
col_shift = 'shift_{}_{}'.format(col, days)
 col_name = 'returns_{}_PrevRaw{}'.format(col, days)
 lag_columns.append(col_name)
 df_code.loc[:, col_shift] = df_code[col].shift(days, freq='D')
 df_code[col_name] =(df_code[col_shift] - df_code[col])/ df_code[col_shift]
df_code.drop(col_shift, axis=1, inplace=True)
 return lag_columns, df_code.drop(features, axis=1)
 @timeit
def define_lagged_var(self):
start_time = time.time()
list_df = [d[1][self.common_lag_features + ['price_diff']].set_index('time' ).copy()
for d in self.history_df.groupby('assetCode', sort=False)]
if self.verbose:
print('assetCodes: {}, time to group: {:<5.2f}s'.format(len(list_df), time.time() - start_time))
grouped = run_parallel(list_df, account_for_past, int(self.verbose))
self.lag_columns = grouped[0][0]
self.merged_df = pd.merge(self.merged_df,
pd.concat([d[1] for d in grouped] ).reset_index() ,
on=self.common_lag_features,
how='left')
 @timeit
def define_lagged_return(self):
start_time = time.time()
list_df = [d[1][self.common_lag_features + ['close']].set_index('time' ).copy()
for d in self.history_df.groupby('assetCode', sort=False)]
if self.verbose:
print('assetCodes: {}, time to group: {:<5.2f}s'.format(len(list_df), time.time() - start_time))
grouped = run_parallel(list_df, create_returns, int(self.verbose))
self.lag_columns = grouped[0][0]
self.merged_df = pd.merge(self.merged_df,
pd.concat([d[1] for d in grouped] ).reset_index() ,
on=self.common_lag_features,
how='left')
 two_sigma_model.define_lagged_var = define_lagged_var.__get__(two_sigma_model)
 two_sigma_model.define_lagged_return = define_lagged_return.__get__(two_sigma_model)
 LAST_DATE_ONLY = False
two_sigma_model.define_lagged_var()
two_sigma_model.define_lagged_return()<merge>
p = [1.3,0.6,0.8,0.5,0.6,0.6]
out = []
for _, row in test.iterrows() :
item = [row['id'], row['prd'], row['lang']]
if(item[2]=='es'):
if(item[1]<0.7):
item[1] *= p[0]
elif(item[2]=='fr'):
if(item[1]<0.7):
item[1] *= p[1]
elif(item[2]=='ru'):
if(item[1]<0.7):
item[1] *= p[2]
elif(item[2]=='it'):
if(item[1]<0.7):
item[1] *= p[3]
elif(item[2]=='tr'):
if(item[1]<0.7):
item[1] *= p[4]
elif(item[2]=='pt'):
if(item[1]<0.7):
item[1] *= p[5]
out.append([item[0], item[1]])
 of = pd.DataFrame(out, columns=['id', 'toxic'])
 print(of.head())
 score1 = roc_auc_score(mybest.toxic.round().astype(int), of.toxic.values)
 score2 = roc_auc_score(of.toxic.round().astype(int), mybest.toxic.values)
 print('%2.4f\t%2.4f'%(100*score1, 100*score2))
@timeit
def news_moving_averages(self, col, window):
col_name = 'news_mva_'+ col + '_' + str(window)+ 'days'
if self.verbose:
print("column '{}' has been created".format(col_name))
avg_col = \
self.history_df[self.common_lag_features+[col]].set_index('time' ).\
groupby('assetCode' ).rolling(window=window, freq='D' ).mean().reset_index()
avg_col.rename(columns = {col: col_name}, inplace = True)
 self.merged_df = pd.merge(self.merged_df,
avg_col[self.common_lag_features + [col_name]],
on=self.common_lag_features,
how='left')
 return col_name
@timeit
def calc_news_moving_averages(self):
self.news_lag_columns = []
for col in ['sentimentNegative_mean']:
new_col = self.news_moving_averages(col, 7)
 self.news_lag_columns.append(new_col)
 two_sigma_model.news_moving_averages = news_moving_averages.__get__(two_sigma_model)
 two_sigma_model.calc_news_moving_averages = calc_news_moving_averages.__get__(two_sigma_model)
<features_selection>
dic = {}
oft = open('/kaggle/input/profanity/Profanity.txt', "r", encoding='utf8')
 for l in oft:
ele = l.strip().lower().split(':')
 dic[ele[0]] = ele[1]
oft.close()
les, lit, ltr, lfr, lru, lpt = 1.2, 1.1, 1.3, 1.2, 1.2, 1.3
of['content'] = test['content']
of['tran'] = test['translated']
of['lang'] = test['lang']
out = []
enpros = dic['en'].split(',')
 for _, row in of.iterrows() :
if(row['lang']=='es'):
lmd = les
elif(row['lang']=='it'):
lmd = lit
elif(row['lang']=='tr'):
lmd = ltr
elif(row['lang']=='fr'):
lmd = lfr
elif(row['lang']=='ru'):
lmd = lru
else:
lmd = lpt
item = [row['id'], row['toxic']]
if(item[1]<0.5):
for w in enpros:
if(str(row['tran'] ).lower().find(w)>=0):
item[1] *= 1.2
break
ws = dic[row['lang']].split(',')
 for w in ws:
if(str(row['content'] ).lower().find(w)>=0):
item[1] *= lmd
break
out.append(item)
 of = pd.DataFrame(out, columns=['id', 'toxic'])
 score1 = roc_auc_score(mybest.toxic.round().astype(int), of.toxic.values)
 score2 = roc_auc_score(of.toxic.round().astype(int), mybest.toxic.values)
 print('%2.4f\t%2.4f'%(100*score1, 100*score2))
<prepare_x_and_y><EOS> 
of['toxic'] = mybest.toxic.values*0.8 + of.toxic.values*0.2
score1 = roc_auc_score(mybest.toxic.round().astype(int), of.toxic.values)
 score2 = roc_auc_score(of.toxic.round().astype(int), mybest.toxic.values)
 print('%2.4f\t%2.4f'%(100*score1, 100*score2))
print(of.head())
 of.to_csv('submission.csv', index=False )