Kaggle competition: Categorical Feature Encoding Challenge II
sum(df.brand_name.isnull() )<count_unique_values>
warnings.filterwarnings('ignore')
 pd.options.display.max_columns = 50
BIN_COL = [f'bin_{i}' for i in range(5)]
NOM_COL = [f'nom_{i}' for i in range(10)]
ORD_COL = [f'ord_{i}' for i in range(6)]
NOM_5_9 = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']
NOM_0_4 = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']
DATE_COL = ['day','month']
df.brand_name.unique().shape<sort_values>
%%time
def read_csv() :
train = pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat-ii/test.csv')
 train_id = train['id']
test_id = test['id']
train.drop('id', axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 return train, test, train_id, test_id
def preprocessing(df):
df.bin_3.replace({'F':0, 'T':1}, inplace=True)
 df.bin_4.replace({'N':0, 'Y':1}, inplace=True)
 ord_1_map = {'Novice':1,'Contributor':2,'Expert':3,'Master':4,'Grandmaster':5}
ord_2_map = {'Freezing':1, 'Cold':2,'Warm':3,'Hot':4, 'Boiling Hot':5,'Lava Hot':6}
df.loc[df['ord_1'].notnull() ,'ord_1'] = df.loc[df['ord_1'].notnull() ,'ord_1'].map(ord_1_map)
 df.loc[df['ord_2'].notnull() ,'ord_2'] = df.loc[df['ord_2'].notnull() ,'ord_2'].map(ord_2_map)
 df.loc[df['ord_3'].notnull() ,'ord_3'] = df.loc[df['ord_3'].notnull() ,'ord_3'].apply(
 lambda c: ord(c)- ord('a')+ 1)
 df.loc[df['ord_4'].notnull() ,'ord_4'] = df.loc[df['ord_4'].notnull() ,'ord_4'].apply(
 lambda c: ord(c)- ord('A')+ 1)
 for col in ['ord_1','ord_2','ord_3','ord_4',]:
df[col] = df[col].astype(np.float32)
 df.day = df.day.replace({3:5,2:6,1:7})
 df.loc[df.ord_5.notnull() , 'ord_5_1'] = df.loc[df.ord_5.notnull() , 'ord_5'].apply(lambda x: x[0])
 df.loc[df.ord_5.notnull() , 'ord_5_2'] = df.loc[df.ord_5.notnull() , 'ord_5'].apply(lambda x: x[1])
 df.loc[df['ord_5_1'].notnull() ,'ord_5_1'] = df.loc[df['ord_5_1'].notnull() ,'ord_5_1'].apply(
 lambda c: ord(c)- ord('a')+ 33 ).astype(np.float32)
 df.loc[df['ord_5_2'].notnull() ,'ord_5_2'] = df.loc[df['ord_5_2'].notnull() ,'ord_5_2'].apply(
 lambda c: ord(c)- ord('a')+ 33)
 return df
def filling_NaN(df):
df.fillna(-1, inplace=True)
 df.day = df.day.astype(int)
 df.month = df.month.astype(int)
 return df
def target_encoding(cols, smoothing=1.0, min_samples_leaf=1):
for col in cols:
encoder = ce.TargetEncoder(cols=col,
smoothing=smoothing,
min_samples_leaf=min_samples_leaf)
 train[f'{col}_mean'] = encoder.fit_transform(train[col], train['target'])[col].astype(np.float32)
 test[f'{col}_mean'] = encoder.transform(test[col])[col].astype(np.float32)
 del encoder
gc.collect()
df.groupby("brand_name" ).agg({"price": "mean"} ).sort_values("price", ascending=False ).head(10 )<count_missing_values>
%%time
USE_DUMMIES = BIN_COL+NOM_COL+DATE_COL+['ord_0','ord_1', 'ord_2','ord_3', 'ord_4']
NOT_DUMMIES = [s for s in train.columns if '_mean' in s]
for col in NOT_DUMMIES:
scaler = MaxAbsScaler()
train[col] = scaler.fit_transform(train[[col]] ).flatten().astype(np.float32)
 test[col] = scaler.transform(test[[col]] ).flatten().astype(np.float32)
 del scaler;gc.collect()
sum(df.price.isnull() )<count_values>
%%time
print('drop features:',test.columns.drop(USE_DUMMIES+NOT_DUMMIES ).tolist())
 target = train['target']
train = train[USE_DUMMIES+NOT_DUMMIES]
test = test[USE_DUMMIES+NOT_DUMMIES]
traintest = pd.concat([train, test], sort=False)
 traintest = traintest.reset_index(drop=True)
 traintest.head(10 )
sum(df.price == 0 )<groupby>
%%time
traintest = pd.get_dummies(traintest,
columns=USE_DUMMIES,
dummy_na=False,
drop_first=False,
sparse=True,
dtype=np.int8)
 traintest.head(10 )
df.groupby("shipping" ).agg({"price": "mean"} )<count_missing_values>
%%time
def convert_sparse(traintest, train_length):
train_ohe = traintest.iloc[:train_length, :]
test_ohe = traintest.iloc[train_length:, :]
train_ohe1 = scipy.sparse.bsr_matrix(train_ohe[NOT_DUMMIES])
 test_ohe1 = scipy.sparse.bsr_matrix(test_ohe[NOT_DUMMIES])
 train_ohe2 = scipy.sparse.csr_matrix(train_ohe.drop(columns=NOT_DUMMIES))
test_ohe2 = scipy.sparse.csr_matrix(test_ohe.drop(columns=NOT_DUMMIES))
train_ohe = scipy.sparse.hstack([train_ohe1,train_ohe2] ).tocsr()
test_ohe = scipy.sparse.hstack([test_ohe1, test_ohe2] ).tocsr()
return train_ohe, test_ohe
train_ohe, test_ohe = convert_sparse(traintest, train_length=len(train))
train_ohe.shape, test_ohe.shape
sum(df.item_description.isnull() )<categorify>
del train, test;gc.collect()
class LabelEncoderPipelineFriendly(LabelEncoder):
def __init__(self, **kwargs):
super(LabelEncoderPipelineFriendly, self ).__init__(**kwargs)
 def fit(self, X, y=None):

 super(LabelEncoderPipelineFriendly, self ).fit(X)
 def transform(self, X, y=None):
return super(LabelEncoderPipelineFriendly, self ).transform(X ).reshape(-1, 1)
 def fit_transform(self, X, y=None):
return super(LabelEncoderPipelineFriendly, self ).fit(X ).transform(X ).reshape(-1, 1 )<categorify>
train_ohe[0:20].todense()
def prepare_data(train, test):
def get_name_col(df):
return df["name"]
def get_condition_col(df):
return df[["item_condition_id"]]
def get_category_col(df):
return df["category_name"].fillna("None" ).astype("category")
 def get_brand_col(df):
return df["brand_name"].fillna("None" ).astype("category")
 def get_shipping_col(df):
return df[["shipping"]]
def get_desc_col(df):
return df["item_description"].fillna("None")
 p = make_union(*[
make_pipeline(FunctionTransformer(get_name_col, validate=False),
TfidfVectorizer(min_df=15)) ,
make_pipeline(FunctionTransformer(get_condition_col, validate=False),
OneHotEncoder()),
make_pipeline(FunctionTransformer(get_category_col, validate=False),
CountVectorizer()),
make_pipeline(FunctionTransformer(get_brand_col, validate=False),
LabelEncoderPipelineFriendly() ,
OneHotEncoder(sparse=True)) ,
make_pipeline(FunctionTransformer(get_shipping_col, validate=False)) ,
make_pipeline(FunctionTransformer(get_desc_col, validate=False),
TfidfVectorizer(ngram_range=(1, 3),
stop_words="english",
max_features=10000))
])
 train_rows = train.shape[0]
df = pd.concat([train, test], axis=0)
 transformed = p.fit_transform(df)
 transformed_train, transformed_test = transformed[:train_rows], transformed[train_rows:]
del df
return(transformed_train, transformed_test )<prepare_x_and_y>
%%time
def print_cv_scores(label, cv_scores):
print(f'{label} cv scores : {cv_scores}')
 print(f'{label} cv mean score : {np.mean(cv_scores)}')
 print(f'{label} cv std score : {np.std(cv_scores)}')
 def run_cv_model(train_ohe, test_ohe, target, model_fn, params={},
eval_fn=None, label='model', cv=5, repeats=5):
if repeats==1:
kf = StratifiedKFold(n_splits=cv, shuffle=True, random_state=42)
 divide_counts = cv
else:
kf = RepeatedStratifiedKFold(n_splits=cv,n_repeats=repeats, random_state=42)
 divide_counts = kf.get_n_splits()
fold_splits = kf.split(train_ohe, target)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros(( train_ohe.shape[0]))
for fold_id,(train_idx, val_idx)in enumerate(fold_splits):
print(f'Started {label} fold:{fold_id} / {divide_counts}')
 tr_X_ohe, val_X_ohe = train_ohe[train_idx], train_ohe[val_idx]
tr_y, val_y = target[train_idx], target[val_idx]
print(Counter(tr_y), Counter(val_y))
params2 = params.copy()
model, pred_val_y, pred_test_y = model_fn(
 tr_X_ohe, tr_y, val_X_ohe, val_y, test_ohe, params2)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_idx] = pred_val_y
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 print(f'{label} cv score {fold_id}: {cv_score}')
 print_cv_scores(label, cv_scores)
 pred_full_test = pred_full_test / divide_counts
results = {'label': label,
'train': pred_train,
'test': pred_full_test,
'cv': cv_scores}
return results
def runLR(train_X, train_y, val_X, val_y, test_X, params):
print('Train LR')
 model = LogisticRegression(**params)
 model.fit(train_X, train_y)
 print('Predict val data')
 pred_val_y = model.predict_proba(val_X)[:, 1]
print('Predict test data')
 pred_test_y = model.predict_proba(test_X)[:, 1]
return model, pred_val_y, pred_test_y
lr_params = {'penalty':'l2',
'solver': 'lbfgs', 'C': 0.05,
'max_iter':500,
'random_state':42,
}
results1 = run_cv_model(
 train_ohe, test_ohe, target, runLR,
lr_params, auc, 'lr', cv=10, repeats=2 )
<compute_test_metric><EOS> 
%%time
submission = pd.DataFrame(
 {'id': test_id, 'target': results1['test'],})
 submission.to_csv('submission.csv', index=False )