Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<set_options>
import os
import numpy as np
import pandas as pd
from scipy.special import softmax
gc.enable()<load_from_csv>
path = ".. /input/jigsaw-multilingual-toxic-comment-classification/"
record = ".. /input/buffer/"
base = record + "submission-9462.csv"
monos = ["submission-it-9467.csv",
"submission-pt-9470.csv",
"submission-es-9467.csv",
"submission-tr-9470.csv",
"submission-fr-9473.csv",]
get_lang = lambda x: x.split('-')[1]
orders = pd.read_csv('.. /input/orders.csv')
 order_products_train = pd.read_csv('.. /input/order_products__train.csv')
 order_products_prior = pd.read_csv('.. /input/order_products__prior.csv')
 products = pd.read_csv('.. /input/products.csv')
 aisles = pd.read_csv('.. /input/aisles.csv')
 departments = pd.read_csv('.. /input/departments.csv' )<data_type_conversions>
test = pd.read_csv(path + "test.csv")
 dic_ids = {k:v.id for k,v in test.groupby(["lang"])}
aisles['aisle'] = aisles['aisle'].astype('category')
 departments['department'] = departments['department'].astype('category')
 orders['eval_set'] = orders['eval_set'].astype('category')
 products['product_name'] = products['product_name'].astype('category' )<merge>
sub = pd.read_csv(base)
 for m in monos:
res = pd.read_csv(record + m)
 ids = dic_ids[get_lang(m)]
sub.loc[ids,"toxic"] = res.toxic[ids]
sub.head()
op = orders.merge(order_products_prior, on='order_id', how='inner')
 op.head()<groupby>
adj = {
"fr":1.04,
"es":1.06,
"pt":.96,
"it":.97,
"tr":.98,
}
for l,v in adj.items() :
ids = dic_ids[l]
sub.loc[ids,"toxic"] *= v
sub.head()
user= orders.groupby('user_id')[['days_since_prior_order']].mean()
user.columns=['avg_days_since_prior_order']
user.head()<prepare_output>
weight = lambda x: softmax(1/(1-x))
def mix_result(subs,pbs):
toxics = np.array([df.toxic.values for df in subs])
 w = weight(np.array(pbs))
print(["{:.3f}".format(i)for i in w])
 return toxics.T@w
user1 = op.groupby('user_id')['order_number'].max().to_frame('u_total_orders')
 user1.head() 
<merge>
sub1 = pd.read_csv(record+"submission-public-mix-9482.csv")
 sub["toxic"] = mix_result([sub,sub1],[.9508,.9482])
 sub.head()
prd = pd.merge(orders, order_products_prior, on='order_id', how='inner')
 prd.head(10 )<prepare_output>
sub1 = pd.read_csv(record+"submission-1st-place-9550.csv")
 sub2 = pd.read_csv(record+"submission-2nd-place-9522.csv")
 sub["toxic"] = mix_result([sub,sub2,sub1],[.9514,.9522,.9550])
 sub.head()
order_size= prd.groupby(['user_id', 'order_id'])['product_id'].count()
order_size = order_size.to_frame()
order_size.columns=['size']
order_size.head()<groupby>
sub.to_csv('submission.csv', index=False )
<merge><EOS> 
sub.to_csv('submission.csv', index=False )