Kaggle competition: Toxic Comment Classification Challenge
train[hh_cont] = train[hh_cont].astype('float64')
 test[hh_cont] = test[hh_cont].astype('float64');<data_type_conversions>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
train[hh_ordered + ind_ordered] = train[hh_ordered + ind_ordered].astype(int)
 test[hh_ordered + ind_ordered] = test[hh_ordered + ind_ordered].astype(int);<data_type_conversions>
from fastai.text import *
from fastai import *
train['Target'] = train['Target'].astype(int);<count_missing_values>
df_train = pd.read_csv(path/'train.csv')
 df_test = pd.read_csv(path/'test.csv')
 df = df_train.append(df_test)
 df.head()
train.isnull().sum() [train.isnull().sum() > 0], test.isnull().sum() [test.isnull().sum() > 0]<feature_engineering>
data_lm.show_batch()
train.loc[train['rent-per-adult'].isnull() , 'rent-per-adult'] = train.loc[train['rent-per-adult'].isnull() , 'v2a1'].values[0]
test.loc[test['rent-per-adult'].isnull() , 'rent-per-adult'] = test.loc[test['rent-per-adult'].isnull() , 'v2a1'].values[0]<count_missing_values>
learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.3 )
train.isnull().sum() [train.isnull().sum() > 0], test.isnull().sum() [test.isnull().sum() > 0]<define_variables>
learn.fit_one_cycle(1, 1e-2, moms=(0.8,0.7))
for c in train.columns:
if train[c].dtype != 'float64': continue
s = np.where(train[c].values >= np.finfo(np.float32 ).max)
 if len(s[0])>0:
print(c)
 print(s )<filter>
learn.save('fit_head' )
train[train['rent-per-adult'] > np.finfo(np.float32 ).max][['rent-per-adult']]<filter>
learn.load('fit_head');
train[train['rent-per-adult'] > np.finfo(np.float32 ).max][['Id', 'idhogar', 'v2a1', 'hogar_adul', 'age']]<filter>
learn.fit_one_cycle(6, 1e-3, moms=(0.8,0.7))
train[train['idhogar']==1959][['idhogar', 'v2a1', 'age']]<filter>
learn.save('fine_tuned' )
train[train['idhogar']==2908][['idhogar', 'v2a1', 'age']]<feature_engineering>
learn.load('fine_tuned');
for h_id in h_ids:
rent_per_adul = train.loc[(train['idhogar']==h_id), 'v2a1'].values[0] / train.loc[(train['idhogar']==h_id)].shape[0]
train.loc[train['idhogar']==h_id, 'rent-per-adult'] = rent_per_adul
train.loc[train['idhogar']==h_id, 'rent-per-adult_sum'] = train.loc[(train['idhogar']==h_id), 'v2a1'].values[0]<filter>
TEXT = "I liked this movie because"
N_WORDS = 40
N_SENTENCES = 2
for c in test.columns:
if test[c].dtype != 'float64': continue
s = np.where(test[c].values >= np.finfo(np.float32 ).max)
 if len(s[0])>0:
print(c)
 print(s )<filter>
print("
".join(learn.predict(TEXT, N_WORDS, temperature=0.75)for _ in range(N_SENTENCES)) )
test[test['rent-per-adult'] > np.finfo(np.float32 ).max][['rent-per-adult']]<filter>
learn.save_encoder('fine_tuned_enc' )
test[test['rent-per-adult'] > np.finfo(np.float32 ).max][['Id', 'idhogar', 'v2a1', 'hogar_adul', 'age']]<feature_engineering>
test = pd.read_csv(path/"test.csv")
 test_datalist = TextList.from_df(test, cols='comment_text')
h_ids = test[test['rent-per-adult'] > np.finfo(np.float32 ).max]['idhogar'].unique()
for h_id in h_ids:
rent_per_adul = test.loc[(test['idhogar']==h_id), 'v2a1'].values[0] / test.loc[(test['idhogar']==h_id)].shape[0]
test.loc[test['idhogar']==h_id, 'rent-per-adult'] = rent_per_adul
test.loc[test['idhogar']==h_id, 'rent-per-adult_sum'] = test.loc[(test['idhogar']==h_id), 'v2a1'].values[0]<create_dataframe>
data_cls =(TextList.from_csv(path, 'train.csv', cols='comment_text', vocab=data_lm.vocab)
.split_by_rand_pct(valid_pct=0.1)
.label_from_df(cols=['toxic', 'severe_toxic','obscene', 'threat', 'insult', 'identity_hate'], label_cls=MultiCategoryList, one_hot=True)
.add_test(test_datalist)
.databunch())
 data_cls.save('data_clas.pkl')
def make_new_features_grouping(df, dtypes, gpby, customAggFunc=None):

 if 'Target' in df.columns: numeric_type = df.select_dtypes(dtypes ).drop(['Target', 'Id'], axis=1 ).copy()
else: numeric_type = df.select_dtypes(dtypes ).drop(['Id'], axis=1 ).copy()
funcs = ['count', 'mean', 'max', 'min', 'sum', 'std', 'var', 'quantile']
if customAggFunc is None: new = numeric_type.groupby(gpby ).agg(funcs)
 elif isinstance(customAggFunc, list): new = numeric_type.groupby(gpby ).agg(funcs + customAggFunc)
 else: new = numeric_type.groupby(gpby ).agg(funcs + [customAggFunc])
 columns = []
for old_col in new.columns.levels[0]:
if old_col != 'idhogar':
for new_col in new.columns.levels[1]:
columns.append(old_col + '_' + new_col)
 new.columns = columns
return df.merge(new.reset_index() , on="idhogar", how='left' )<drop_column>
data_clas = load_data(path, 'data_clas.pkl', bs=bs )
%time train = make_new_features_grouping(train, ['number', bool], "idhogar")
 %time test = make_new_features_grouping(test, ["number", bool], "idhogar" )<train_model>
data_clas.show_batch()
train.fillna(0, inplace=True)
 test.fillna(0, inplace=True )<normalization>
learn = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5)
 learn.load_encoder('fine_tuned_enc' )
scaler1 = RobustScaler()
scaler2 = RobustScaler()
scaled1 = scaler1.fit_transform(train.drop(['Target', 'Id', 'idhogar'], axis=1))
scaled2 = scaler2.fit_transform(test.drop(['Id', 'idhogar'], axis=1))
cols1 = train.drop(['Target', 'Id', 'idhogar'], axis=1 ).columns
cols2 = test.drop(['Id', 'idhogar'], axis=1 ).columns
trPCA = pd.DataFrame(scaled1, index=np.arange(train.shape[0]), columns=cols1)
 tsPCA = pd.DataFrame(scaled2, index=np.arange(test.shape[0]), columns=cols2 )<choose_model_class>
learn.fit_one_cycle(1, 2e-2, moms=(0.8,0.7))
pca = PCA(n_components=5, svd_solver='full' )<categorify>
learn.save('first' )
transformed1 = pca.fit_transform(trPCA)
 transformed2 = pca.transform(tsPCA)
 for i in range(5):
train[f'PCA{i+1}'] = transformed1[:,i]
test[f'PCA{i+1}'] = transformed2[:,i]
train.shape, test.shape<set_options>
learn.load('first');
pca.explained_variance_ratio_<count_values>
learn.freeze_to(-2)
 learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7))
train['Target'].value_counts() , 774+1221+1558+1500<split>
learn.save('second');
learn.load('second');
def train_val_split(rows, tvlen=None, vper=None):

 if tvlen is not None and vper is None:
count1 = tvlen[0]
count2 = tvlen[1]
elif tvlen is None and vper is not None:
c = rows.sum()
count1 = int(( 1-vper)*c)
 count2 = int(vper*c)
 else:
raise Exception('One of "tvlen" or "vper" should be given.')
 rowst, rowsv = rows.copy() , rows.copy()
for i in range(len(rows)) :
if not count1:
rowst[i] = False
if not count2:
rowsv[i] = False
count2 -= rowsv[i]
continue
rowsv[i] = False
count1 -= rowst[i]
return rowst, rowsv<split>
learn.freeze_to(-3)
 learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))
rows123t, rows123v = train_val_split(rows123, vper=0.1)
 rows4t, rows4v = train_val_split(rows4, tvlen=(1300, 200))
rows123t.sum() , rows123v.sum() , rows4t.sum() , rows4v.sum()<drop_column>
learn.save('third')
 learn.load('third');
train.drop(['Id', 'idhogar'], axis=1, inplace=True )<prepare_x_and_y>
learn.unfreeze()
learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))
xtrain, xvalid = train.loc[rows123t|rows4t].drop('Target', axis=1 ).copy() , train.loc[rows123v|rows4v].drop('Target', axis=1 ).copy()
ytrain, yvalid = train['Target'].loc[rows123t|rows4t].copy() , train['Target'].loc[rows123v|rows4v].copy()<count_values>
learn.predict("I really loved that movie, it was awesome!" )
ytrain.value_counts()<count_values>
preds, target = learn.get_preds(DatasetType.Test, ordered=True)
 labels = preds.numpy()
yvalid.value_counts()<count_values>
test_id = test['id']
label_cols = ['toxic', 'severe_toxic' , 'obscene' , 'threat' , 'insult' , 'identity_hate']
submission = pd.DataFrame({'id': test_id})
 submission = pd.concat([submission, pd.DataFrame(preds.numpy() , columns = label_cols)], axis=1)
 submission.to_csv('submission.csv', index=False)
 submission.head()