Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<load_from_csv>
import pandas as pd
import numpy as np
import category_encoders as ce
import lightgbm as lgb
from sklearn import linear_model
from sklearn.model_selection import StratifiedKFold
import gc
submission = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv', index_col='id')
 submission['prediction'] = model.predict(pad_text(test[TEXT_COLUMN], tokenizer))
submission.reset_index(drop=False, inplace=True)
 submission.head()<save_to_csv>
warnings.filterwarnings("ignore", category=DeprecationWarning)
 warnings.filterwarnings("ignore", category=UserWarning)
 warnings.filterwarnings("ignore", category=FutureWarning)
submission.to_csv('submission.csv', index=False )<import_modules>
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv')
 print(train.target.value_counts() [0]/300000, train.target.value_counts() [1]/300000,)
 train.sort_index(inplace=True)
 train_y = train['target']
test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True )
import numpy as np
import pandas as pd
from kaggle.competitions import twosigmanews
from datetime import datetime, date
import numpy as np
from sklearn import model_selection 
<prepare_output>
HTML('<iframe width="680" height="620" src="https://www.youtube.com/embed/8odLEbSGXoI" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>')
env = twosigmanews.make_env() 
(market_train_orig, news_train_orig)= env.get_training_data()
market_train_df = market_train_orig.copy()
news_train_df = news_train_orig.copy()<feature_engineering>
cat_feat_to_encode = train.columns.tolist()
smoothing=0.50
oof = pd.DataFrame([])
 for tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state=1, shuffle=True ).split(train, train_y):
ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train.iloc[tr_idx, :], train_y.iloc[tr_idx])
 oof = oof.append(ce_target_encoder.transform(train.iloc[oof_idx, :]), ignore_index=False)
 ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train, train_y)
 train = oof.sort_index()
test = ce_target_encoder.transform(test )
market_train_df['time'] = market_train_df['time'].dt.date
market_train_df = market_train_df.loc[market_train_df['time']>=date(2009, 1, 1)]<feature_engineering>
glm = linear_model.LogisticRegression(
 random_state=1, solver='lbfgs', max_iter=2019, fit_intercept=True,
penalty='none', verbose=0)
 glm.fit(train, train_y )
news_train_df['time'] = news_train_df['time'].dt.date
news_train_df = news_train_df.loc[news_train_df['time']>=date(2009, 1, 1)]<drop_column>
pd.DataFrame({'id': test_id, 'target': glm.predict_proba(test)[:,1]} ).to_csv(
 'sub_' + str(datetime.now().strftime('%Y-%m-%d_%H-%M-%S')) + '.csv',
index=False )