Kaggle competition: Toxic Comment Classification Challenge
def preprocess_calendar(calendar):
global maps, mods
calendar["event_name"] = calendar["event_name_1"]
calendar["event_type"] = calendar["event_type_1"]
map1 = {mod:i for i,mod in enumerate(calendar['event_name'].unique())}
calendar['event_name'] = calendar['event_name'].map(map1)
 map2 = {mod:i for i,mod in enumerate(calendar['event_type'].unique())}
calendar['event_type'] = calendar['event_type'].map(map2)
 calendar['nday'] = calendar['date'].str[-2:].astype(int)
 maps["event_name"] = map1
maps["event_type"] = map2
mods["event_name"] = len(map1)
 mods["event_type"] = len(map2)
 calendar["wday"] -=1
calendar["month"] -=1
calendar["year"] -= 2011
mods["month"] = 12
mods["year"] = 6
mods["wday"] = 7
mods['snap_CA'] = 2
mods['snap_TX'] = 2
mods['snap_WI'] = 2
calendar.drop(["event_name_1", "event_name_2", "event_type_1", "event_type_2", "date", "weekday"],
axis=1, inplace=True)
 return calendar 
<load_from_csv>
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv' )
maps, mods = {}, {}
calendar = pd.read_csv(".. /input/m5-forecasting-uncertainty/calendar.csv")
 calendar = preprocess_calendar(calendar)
 calendar['nb'] = calendar.index + 1
calendar['n_week'] =(calendar.index - 2)// 7<categorify>
value_counts = train_data.iloc[:,2:].apply(pd.Series.value_counts)
 print(value_counts.iloc[1])
%%time
FE = ['snap_CA','snap_TX','snap_WI']
for col in tqdm(['wday','month','year','event_name','nday']):
_temp = pd.get_dummies(calendar[col], prefix=col)
 new_cols = list(_temp.columns)
 FE += new_cols
calendar = calendar.join(_temp)
<load_from_csv>
word_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
max_features=30000)
 char_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='char',
ngram_range=(1, 4),
max_features=30000)
sales = pd.read_csv(".. /input/walmartadd/sales_aug.csv" )<define_variables>
vectorizer = make_union(word_vectorizer, char_vectorizer, n_jobs=3 )
START = 1000
H = 28
B = 28
COLS = [f"d_{i}" for i in range(START, 1942)]
calendar = calendar[calendar.nb>=START]
ids = sales.id.values<find_best_params>
train_comments = train_data['comment_text']
test_comments = test_data['comment_text']
vectorizer.fit(train_comments)
 train_features = vectorizer.transform(train_comments)
 test_features = vectorizer.transform(test_comments )
<feature_engineering><EOS> 
scores = []
class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
submission = pd.DataFrame.from_dict({'id': test_data['id']})
 for class_name in class_names:
train_target = train_data[class_name]
classifier = LogisticRegression(solver='sag')
 cv_score = np.mean(cross_val_score(
 classifier, train_features, train_target, cv=3, scoring='roc_auc'))
scores.append(cv_score)
 print('CV score for class {} is {}'.format(class_name, cv_score))
classifier.fit(train_features, train_target)
 submission[class_name] = classifier.predict_proba(test_features)[:, 1]
print('Total CV score is {}'.format(np.mean(scores)))
 submission.to_csv('submission.csv', index=False )