Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<define_variables>
data = {k.split('/')[-1][:-4]:k for k in glob.glob('/kaggle/input/**/**.csv')}
train = pd.read_csv(data['jigsaw-toxic-comment-train'], usecols=['id', 'comment_text', 'toxic'])
 val = pd.read_csv(data['validation'], usecols=['comment_text', 'toxic'])
 test = pd.read_csv(data['test'], usecols=['id', 'content'])
 test.columns = ['id', 'comment_text']
test['toxic'] = 0.5
sub2 = pd.read_csv('.. /input/finalsubmission/submission-.9480.csv')
 sub4 = pd.read_csv('.. /input/finalsubmission/submission-.9481.csv' )
days = env.get_prediction_days()<concatenate>
%%time
def f_experience(c, s):
it = {'memory':10,
'influence':0.5,
'inference':0.5,
'interest':0.9,
'sentiment':1e-10,
'harmony':0.5}
exp = {}
for i in range(len(c)) :
words = set([w for w in str(c[i] ).lower().split(' ')])
 for w in words:
try:
exp[w]['influence'] = exp[w]['influence'][1:] + [s[i]]
exp[w]['inference'] += 1
exp[w]['interest'] = exp[w]['interest'][1:] + [(exp[w]['interest'][it['memory']-1] +(s[i] * it['interest'])) /2]
exp[w]['sentiment'] += s[i]
except:
m = [0.for m_ in range(it['memory'])]
exp[w] = {}
exp[w]['influence'] = m[1:] + [s[i]]
exp[w]['inference'] = 1
exp[w]['interest'] = m[1:] + [s[i] * it['interest'] / 2]
exp[w]['sentiment'] = s[i]
for w in exp:
exp[w]['sentiment'] /= exp[w]['inference'] + it['sentiment']
exp[w]['inference'] /= len(c)* it['inference']
return exp
exp = f_experience(train['comment_text'].values, train['toxic'].values )
@timeit
def add_new_date(self):
self.history_df = pd.concat([self.history_df,
self.merged_df[
self.common_lag_features +
self.used_for_lag +
self.used_for_news_lag]])
 if self.verbose:
print("data for lagged quantities from {} to {}".format(
 self.lag_df['time'].min().date() ,
self.lag_df['time'].max().date()))
two_sigma_model.add_new_date = add_new_date.__get__(two_sigma_model )<predict_on_test>
%%time
def features(df):
df['len'] = df['comment_text'].map(len)
 df['wlen'] = df['comment_text'].map(lambda x: len(str(x ).split(' ')))
 df['influence_sum'] = df['comment_text'].map(lambda x: np.sum([np.mean(exp[w]['influence'])if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['influence_mean'] = df['comment_text'].map(lambda x: np.mean([np.mean(exp[w]['influence'])if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['inference_sum'] = df['comment_text'].map(lambda x: np.sum([exp[w]['inference'] if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['inference_mean'] = df['comment_text'].map(lambda x: np.mean([exp[w]['inference'] if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['interest_sum'] = df['comment_text'].map(lambda x: np.sum([np.mean(exp[w]['interest'])if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['interest_mean'] = df['comment_text'].map(lambda x: np.mean([np.mean(exp[w]['interest'])if w in exp else 0 for w in str(x ).lower().split(' ')]))
df['sentiment_sum'] = df['comment_text'].map(lambda x: np.sum([exp[w]['sentiment'] if w in exp else 0.5 for w in str(x ).lower().split(' ')]))
df['sentiment_mean'] = df['comment_text'].map(lambda x: np.mean([exp[w]['sentiment'] if w in exp else 0.5 for w in str(x ).lower().split(' ')]))
return df
val = features(val)
 test= features(test )
@timeit
def predict(self, date, pred_template):
df = self.merged_df[self.merged_df['time'] == date]
predictions = self.model.predict(
 df[self.selected_variables].values,
ntree_limit = self.num_boost_round 
)
 preds = pd.DataFrame({'assetCode': df['assetCode'],
'confidence': np.clip(predictions, -1, 1)})
 self.formated_pred = \
pred_template.merge(preds, how='left' ).\
drop('confidenceValue', axis=1 ).\
fillna(0 ).\
rename(columns={'confidence': 'confidenceValue'})
 two_sigma_model.predict = predict.__get__(two_sigma_model )<filter>
col = [c for c in val if c not in ['id', 'comment_text', 'toxic']]
x1, x2, y1, y2 = model_selection.train_test_split(val[col], val['toxic'], test_size=0.3, random_state=20)
 model = ensemble.ExtraTreesClassifier(n_estimators=1000, max_depth=7, n_jobs=-1, random_state=20)
 model.fit(x1, y1)
 print(metrics.roc_auc_score(y2, model.predict_proba(x2)[:,1].clip(0.,1.)))
model.fit(val[col], val['toxic'])
 test['toxic'] = model.predict_proba(test[col])[:,1].clip(0.,1.)
sub1 = test[['id', 'toxic']]
two_sigma_model.history_df = \
two_sigma_model.history_df[two_sigma_model.history_df['time'] >(two_sigma_model.history_df['time'].max() - timedelta(30)) ]<predict_on_test>
sub1.rename(columns={'toxic':'toxic1'}, inplace=True)
 sub2.rename(columns={'toxic':'toxic2'}, inplace=True)
 sub4.rename(columns={'toxic':'toxic4'}, inplace=True)
 sub3 = sub1.merge(sub2,on='id' ).merge(sub4,on='id' )
TIME_FUNCTIONS = False
n_days = 0
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
start_time = time.time()
print(n_days, end=' ')
 two_sigma_model.data_prep(market_obs_df, news_obs_df)
 date = two_sigma_model.merged_df['time'].max()
two_sigma_model.add_new_date()
two_sigma_model.define_lagged_var()
two_sigma_model.define_lagged_return()
two_sigma_model.select_variables()
two_sigma_model.predict(date, predictions_template_df)
 env.predict(two_sigma_model.formated_pred )<save_to_csv>
sub3['toxic'] =(sub3['toxic1'] * 0.1)+(sub3['toxic4'] * 0.9)
 sub3['toxic'] =(sub3['toxic2'] * 0.49)+(sub3['toxic'] * 0.51)
 sub3[['id', 'toxic']].to_csv('submission.csv', index=False )
<import_modules><EOS> 
test = pd.DataFrame(['Howling with Wolf on Lügenpresse'], columns=['comment_text'])
 test['id'] = test.index
test= features(test)
 test['toxic'] = model.predict_proba(test[col])[:,1].clip(0.,1.)
test[['id', 'comment_text', 'toxic']].head()