Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<data_type_conversions>
DATA_PATH = "/kaggle/input/jigsaw-multilingual-toxic-comment-classification"
val_df = pd.read_csv(os.path.join(DATA_PATH, "validation.csv"))
test_df = pd.read_csv(os.path.join(DATA_PATH, "test.csv"))
test_en_df = pd.read_csv(".. /input/test-en-df/test_en.csv")
 val_ids_df = pd.read_csv(os.path.join(DATA_PATH, "validation-processed-seqlen128.csv"))
test_ids_df = pd.read_csv(os.path.join(DATA_PATH, "test-processed-seqlen128.csv"))
test_ids_df = pd.merge(test_ids_df, test_df[["id", "lang"]], on="id" )
last_five['uxp_ratio_last_five'] = last_five.uxp_last_five / 5.0
last_five.head()<data_type_conversions>
target_column = "comment_text"
test_ids_df.loc[test_ids_df[target_column].isin(val_ids_df[target_column]), ]
<merge>
target_column = "input_word_ids"
overlapped_test_df = test_ids_df.loc[test_ids_df[target_column].isin(val_ids_df[target_column]), ]
overlapped_val_df = val_ids_df.loc[val_ids_df[target_column].isin(test_ids_df[target_column]), ]
overlapped_test_df.nunique()
uxp = uxp.merge(last_five, on=['user_id', 'product_id'], how='left')
 del [last_five, temp]
gc.collect()
uxp.head()<correct_missing_values>
test_ids_df["token_hash"] = test_ids_df["input_word_ids"].apply(lambda x: crc32(x.encode())& 0xffffffff)
 val_ids_df["token_hash"] = val_ids_df["input_word_ids"].apply(lambda x: crc32(x.encode())& 0xffffffff)
 target_column = "token_hash"
overlapped_hash = test_ids_df.loc[test_ids_df[target_column].isin(val_ids_df[target_column]), target_column]
overlapped_hash = np.unique(overlapped_hash.values)
 overlapped_hash.sort()
overlapped_hash.shape
uxp.fillna(0, inplace=True)
 uxp.head(10 )<merge>
def display_test_val_comment(test_ids_df, val_ids_df, target_column="token_hash", target_value=0, test_en_df=None, ind_=0):
test_sample = test_ids_df.query(f"{target_column} == {target_value}")
 val_sample = val_ids_df.query(f"{target_column} == {target_value}")
 test_comment = test_sample.comment_text.values[ind_]
val_comment = val_sample.comment_text.values[0]
print("{}".format("-"*80))
print(f"{target_column}:{target_hash}")
 print(f">> TEST
ID:{test_sample.id.values[ind_]}, LANG:{test_sample.lang.values[ind_]}, Duplicated Num:{len(test_sample)}")
 print(f"COMMET_TEXT:{test_comment}
")
 print(f">> VALIDATION
ID:{val_sample.id.values[0]}, LANG:{val_sample.lang.values[0]}, Duplicated Num:{len(val_sample)}, TOXIC:{val_sample.toxic.values[0]}")
 print(f"COMMET_TEXT:{val_comment}
")
 " ).split() , val_comment.replace(" ", "
" ).split() , "TEST", "VAL", lineterm='
')
 diff = difflib.context_diff(test_comment.replace(" ", "
" ).split() , val_comment.replace(" ", "
" ).split() , "TEST", "VAL", lineterm='
')
 print(">> diff result")
 sys.stdout.writelines(diff)
 if test_en_df is not None:
test_en_comment = test_en_df.query(f"id == {test_sample.id.values[ind_]}" ).content_en.values[0]
print("
>> TEST ENGLISH Translation")
 print(f"{test_en_comment}
" )
data = uxp.merge(users, on='user_id', how='left')
 data.head()<merge>
for i, target_hash in enumerate(overlapped_hash):
display_test_val_comment(test_ids_df, val_ids_df, target_column="token_hash", target_value=target_hash, test_en_df=test_en_df)
 if i >= 2:
break
data = data.merge(prd, on='product_id', how='left')
 data.head()<drop_column>
sub_df = pd.read_csv(".. /input/ensemble/submission.csv")
 sub_df = pd.merge(sub_df, test_ids_df, on="id")
 sub_df.head()
del [users, prd, uxp]
gc.collect()<filter>
sub_df.loc[sub_df.token_hash.isin(non_toxic_hash), :].sort_values("toxic" ).tail()
orders_future = orders.loc[(( orders.eval_set == 1)|(orders.eval_set == 2)) , ['user_id', 'eval_set', 'order_id']]
orders_future.head()<merge>
non_toxic_hash = non_toxic_hash[non_toxic_hash != 3880127965]
data = data.merge(orders_future, on='user_id', how='left')
 data.head()<split>
sub_df.loc[sub_df.token_hash.isin(non_toxic_hash), "toxic"] = 0.0
sub_df.loc[sub_df.token_hash.isin(toxic_hash), "toxic"]= 1.0
data_train = data[data.eval_set == 1]
data_train.head()<merge>
sub_df[["id", "toxic"]].to_csv("submission.csv", index=False )
data_train = data_train.merge(order_products_train[['product_id', 'order_id', 'reordered']], on=['product_id', 'order_id'], how='left')
 data_train.head()<drop_column>
test_ids_df.duplicated("input_word_ids", keep=False ).sum()
data_train = data_train.set_index(['user_id', 'product_id'])
 data_train.drop(['eval_set', 'order_id'], axis=1, inplace=True )<split>
val_ids_df.duplicated("input_word_ids", keep=False ).sum()