Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<define_variables>
import numpy as np
import pandas as pd
import category_encoders as ce
from sklearn.metrics import roc_auc_score
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold
print('Kept %d words.' % len(name_vec.get_feature_names()))<categorify>
train = pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat-ii/test.csv' )
desc_vec = TfidfVectorizer(max_features=100000,
stop_words=[stopwords, string.punctuation])
 desc_matrix = desc_vec.fit_transform(df.item_description)
 desc_matrix_sub= desc_vec.transform(df_sub.item_description )<categorify>
train.sort_index(inplace=True)
 train_y = train['target']; test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True); test.drop('id', axis=1, inplace=True )
cond_matrix = sparse.csr_matrix(pd.get_dummies(df.item_condition_id, sparse=True, drop_first=True))
cond_matrix_sub = sparse.csr_matrix(pd.get_dummies(df_sub.item_condition_id, sparse=True, drop_first=True))<concatenate>
folds = 20
smoothing=100
cat_feat_to_encode = train.columns[:].tolist()
oof = np.zeros(train.shape)
 test_oof = np.zeros(test.shape)
 for tr_idx, oof_idx in StratifiedKFold(n_splits=folds, random_state= 1032, shuffle=True ).split(train, train_y):
ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train.iloc[tr_idx, :], train_y.iloc[tr_idx])
 oof[oof_idx,:] = ce_target_encoder.transform(train.iloc[oof_idx, :] ).values
test_oof[:,:] += ce_target_encoder.transform(test ).values
test_oof /= folds
ship_matrix = sparse.csr_matrix(df.shipping ).transpose()
ship_matrix_sub = sparse.csr_matrix(df_sub.shipping ).transpose()<concatenate>
train = pd.DataFrame(data=oof,columns=cat_feat_to_encode,index=train.index.values)
 test = pd.DataFrame(data=test_oof,columns=cat_feat_to_encode,index=test.index.values)
sparse_matrix = sparse.csr_matrix(sparse.hstack([cat_matrix, brand_matrix, name_matrix, desc_matrix,
cond_matrix, ship_matrix]))
sparse_matrix_sub = sparse.csr_matrix(sparse.hstack([cat_matrix_sub, brand_matrix_sub, name_matrix_sub,
desc_matrix_sub, cond_matrix_sub, ship_matrix_sub]))<train_model>
glm =LogisticRegression(random_state=2, solver='lbfgs', max_iter=20600, fit_intercept=True, penalty='l2', verbose=0)
 glm.fit(train, train_y )
if sparse_matrix.shape[1] == sparse_matrix_sub.shape[1]:
print('Features check out.')
 else:
print("The number of features in training and test set don't match." )<drop_column>
roc_auc_score(train_y,glm.predict_proba(train)[:,1] )
<compute_test_metric><EOS> 
pd.DataFrame({'id': test_id, 'target': glm.predict_proba(test)[:,1]} ).to_csv('submission.csv', index=False )