Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<save_to_csv>
!pip install fastai2 --quiet
test_df = pd.read_csv(".. /input/test.csv")
 classifier.fit(x=X, y=y, max_steps=5500)
 test_df = pd.get_dummies(test_df)
 X_test = test_df[features]
id_ = test_df["id"]
pred = classifier.predict(X_test)
 pred = letype.inverse_transform(pred)
 output = pd.DataFrame({"id": id_,"type":pred})
 output.to_csv('ghost_after_getdummies.csv',index=False )<import_modules>
from fastai2.text.all import *
import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.svm import SVR
from sklearn.linear_model import ElasticNet
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from scipy.stats import uniform, randint<load_from_csv>
path = Path('.. /input/jigsaw-toxic-comment-classification-challenge' )
df_train = pd.read_csv('/kaggle/input/mercedes-benz-greener-manufacturing/train.csv.zip')
 df_test = pd.read_csv('/kaggle/input/mercedes-benz-greener-manufacturing/test.csv.zip')
 submission = pd.read_csv('/kaggle/input/mercedes-benz-greener-manufacturing/sample_submission.csv.zip' )<drop_column>
with ZipFile(path/'train.csv.zip', 'r')as zip_ref:
zip_ref.extractall('.. /output/kaggle/working')
 with ZipFile(path/'test.csv.zip', 'r')as zip_ref:
zip_ref.extractall('.. /output/kaggle/working')
 with ZipFile(path/'test_labels.csv.zip', 'r')as zip_ref:
zip_ref.extractall('.. /output/kaggle/working')
 with ZipFile(path/'sample_submission.csv.zip', 'r')as zip_ref:
zip_ref.extractall('.. /output/kaggle/working' )
df_test = df_test.drop(columns=columns)
 df_test<categorify>
path_w = Path('.. /output/kaggle/working' )
for column in df_train:
if df_train[column].dtypes == 'object':
X = [df_train[column], df_test[column]]
X = np.hstack(X)
 enc = LabelEncoder()
enc.fit(X)
 df_train[column] = enc.transform(df_train[column])
 df_test[column] = enc.transform(df_test[column])
 df_train<prepare_x_and_y>
df = pd.read_csv(path_w/'train.csv' )
X_train = df_train.loc[:, 'X0':]
y_train = df_train.loc[:, 'y']<train_on_grid>
blocks =(TextBlock.from_df(text_cols='comment_text', is_lm=True, res_col_name='text'))
param_random = {
'learning_rate': uniform(0.01, 0.1),
'n_estimators': randint(100, 300),
'max_depth': randint(3, 5)
 }
model_gbr = RandomizedSearchCV(GradientBoostingRegressor() , param_random, n_jobs=-1)
 model_gbr.fit(X_train, y_train)
 model_gbr.best_score_<train_on_grid>
test_df = pd.read_csv(path_w/'test.csv' )
param_random = {
'C': uniform(0.1, 3.0),
}
model_svr = RandomizedSearchCV(SVR() , param_random, n_jobs=-1)
 model_svr.fit(X_train, y_train)
 model_svr.best_score_<train_on_grid>
text_df = pd.Series.append(df['comment_text'], test_df['comment_text'] )
param_random = {
'alpha': uniform(0.01, 100),
'l1_ratio': uniform(0.25, 0.75)
 }
model_esn = RandomizedSearchCV(ElasticNet() , param_random, n_jobs=-1, n_iter=100)
 model_esn.fit(X_train, y_train)
 model_esn.best_score_<save_to_csv>
text_df = pd.DataFrame(text_df )
X_test = df_test.loc[:, 'X0':]
y_test = 0.5 * model_gbr.predict(X_test)+ 0.1 * model_svr.predict(X_test)+ 0.4 * model_esn.predict(X_test)
 submission['y'] = y_test
submission.to_csv('submssion.csv', index=False )<set_options>
get_x = ColReader('text')
 splitter = RandomSplitter(0.1, seed=42 )
warnings.filterwarnings("ignore")
 pd.set_option('display.max_rows', 1000)
 rcParams['figure.figsize'] = 8, 5
SEED=2020
kf = KFold(n_splits = 5, random_state = 2)
 InteractiveShell.ast_node_interactivity = "all" 
<load_from_csv>
lm_dblock = DataBlock(blocks=blocks,
get_x=get_x,
splitter=splitter )
train= pd.read_csv('/kaggle/input/mercedes-benz-greener-manufacturing/train.csv.zip')
 train.sample(20 )<count_unique_values>
lm_learn = language_model_learner(lm_dls, AWD_LSTM, pretrained=True, metrics=[accuracy, Perplexity() ] )
one_value_cols = [col for col in train.columns if train[col].nunique() <= 1]
print(f'There are {len(one_value_cols)} columns in train dataset with one unique value.' )<feature_engineering>
lm_learn.to_fp16()
lm_learn.fine_tune(10, 4e-3 )
def basic_details(df):
b = pd.DataFrame()
b['Missing value'] = df.isnull().sum()
b['N unique value'] = df.nunique()
b['dtype'] = df.dtypes
return b
basic_details(train )<drop_column>
lm_learn.save_encoder('fine_tuned' )
train.drop(one_value_cols, axis=1, inplace=True)
<drop_column>
ys = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult',
'identity_hate']
variance_treshold = 0.9
low_var_cols = [col for col in train.columns if train[col].value_counts(dropna=False, normalize=True ).values[0] > variance_treshold]
train=train.drop(low_var_cols, axis=1 )<data_type_conversions>
blocks =(TextBlock.from_df('comment_text', seq_len=lm_dls.seq_len, vocab=lm_dls.vocab),
MultiCategoryBlock(encoded=True, vocab=ys))
def feature_creation(df):
for i in ['X0', 'X1', 'X2', 'X3', 'X5', 'X6', 'X8']:
for j in ['X0', 'X1', 'X2', 'X3', 'X5', 'X6', 'X8']:
df[i + "_" + j] = df[i].astype('str')+ "_" + df[j].astype('str')
 return df
train = feature_creation(train )<categorify>
dls = toxic_clas.dataloaders(df )
lencoders = {}
for col in train.select_dtypes(include=['object'] ).columns:
lencoders[col] = LabelEncoder()
train[col] = lencoders[col].fit_transform(train[col])
<remove_duplicates>
loss_func = BCEWithLogitsLossFlat(thresh=0.8)
 metrics = [partial(accuracy_multi, thresh=0.8)]
train = train.drop_duplicates() 
<filter>
learn = text_classifier_learner(dls, AWD_LSTM, metrics=metrics, loss_func=loss_func )
train = train[(train['y'] <= 136)].reset_index(drop=True )<feature_engineering>
learn.lr_find()
train["y"]=(( train["y"]-train["y"].min())/(train["y"].max() -train["y"].min()))<normalization>
learn.load_encoder('fine_tuned');
X = train.drop('y', axis=1)
 y = train.y
X['n0'] =(X == 0 ).sum(axis=1)
 scaler = preprocessing.MinMaxScaler()
X = pd.DataFrame(scaler.fit_transform(X), columns = X.columns)
 X.sample(10 )<compute_train_metric>
learn.to_fp16()
lr = 1e-2
moms =(0.8,0.7, 0.8)
 lr *= learn.dls.bs/128
learn.fit_one_cycle(1, lr, moms=moms, wd=0.1 )
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=SEED)
 rf = RandomForestRegressor(max_depth=4, n_estimators=5, random_state=SEED)
 cv = KFold(n_splits=10, shuffle=True, random_state=SEED)
 scores = cross_val_score(rf, X, y, scoring='r2', cv=cv, n_jobs=-1)
 s_mean = mean(scores)
 print('Mean R2: %.3f' %(s_mean))<train_on_grid>
learn.freeze_to(-2)
 lr/=2
learn.fit_one_cycle(1, slice(lr/(2.6**4), lr), moms=moms, wd=0.1 )
regressor = xgb.XGBRegressor()
selector = RFECV(regressor, step = 1, cv=cv, n_jobs=-1,verbose=1, scoring='r2')
 selector.fit(X, y)
 print('The optimal number of features is {}'.format(selector.n_features_))
features_rfecv = [f for f,s in zip(X, selector.support_)if s]
print('The selected features are:')
 print('{}'.format(features_rfecv))<compute_train_metric>
learn.freeze_to(-3)
 lr /=2
learn.fit_one_cycle(1, slice(lr/(2.6**4), lr), moms=moms, wd=0.1 )
rf = RandomForestRegressor(max_depth=3, n_estimators=10, random_state=SEED)
 X_rfe = X[features_rfecv]
rf.fit(X_rfe, y)
 scores = cross_val_score(rf, X_rfe, y, cv=cv, scoring = 'r2')
 print("RF based on selected dataset")
 print("FR CV Accuracy Score after selection:", scores.mean().round(3))<train_model>
learn.unfreeze()
lr /= 5
learn.fit_one_cycle(2, slice(lr/(2.6**4),lr), moms=(0.8,0.7,0.8), wd=0.1 )
rf.fit(X_rfe, y)
 perm = PermutationImportance(rf, random_state=SEED ).fit(X_rfe, y)
 eli5.show_weights(perm, feature_names = X_rfe.columns.tolist() )<compute_test_metric>
dl = learn.dls.test_dl(test_df['comment_text'] )
def mae(y_true, y_pred):
return np.mean(abs(y_true - y_pred))
baseline_guess = np.median(y)
 print('The baseline guess based on Y-median value %0.2f' % baseline_guess)
 print("Baseline Performance based on Y-median value: MAE = %0.4f" % mae(y_test, baseline_guess))
print("Baseline Performance based on Random Forest RFECV-model: MAE = %0.4f" % mae(y_test, rf.predict(X_test[features_rfecv])) )<install_modules>
preds = learn.get_preds(dl=dl )
!pip install category_encoders<load_from_csv>
sub = pd.read_csv(path_w/'sample_submission.csv' )
df_train = pd.read_csv(".. /input/train.csv.zip")
 df_test = pd.read_csv(".. /input/test.csv.zip")
 print("Train shape : ", df_train.shape)
 print("Test shape : ", df_test.shape)
 df_train.head()<categorify>
preds[0][0].cpu().numpy()
<categorify>
sub[ys] = preds[0]
for column in ['X0', 'X1', 'X2', 'X3', 'X4', 'X5', 'X6', 'X8']:
le = preprocessing.LabelEncoder()
le.fit(list(df_train[column])+ list(df_test[column]))
df_train[column] = le.transform(df_train[column])
 df_test[column] = le.transform(df_test[column])
 target_columns = df_train.columns[2:]
train_onehot = df_train[target_columns]
test_onehot = df_test[target_columns]<categorify>
sub.to_csv('submission.csv', index=False )
<prepare_x_and_y><EOS> 
sub.to_csv('submission.csv', index=False )