Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<categorify>
%matplotlib inline
test_features = pd.read_csv(".. /input/cat-in-the-dat-ii/test.csv")
 train_set = pd.read_csv(".. /input/cat-in-the-dat-ii/train.csv")
 train_targets = train_set.target
train_features = train_set.drop(['target'], axis=1)
 percentage = train_targets.mean() * 100
print("The percentage of ones in the training target is {:.2f}%".format(percentage))
train_features.head()
start_time = time.time()
print('Encoding of item_condition_id and shipping and start time is:' , 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( start_time))))))
combdata['hot_shp_item_cnd'] = csr_matrix(pd.get_dummies(combdata[["item_condition_id", "shipping" ]], sparse=True ).values)
 print(' Shape of X_shp_item_cnd after encoding %s...'%(str(combdata['hot_shp_item_cnd'].shape)))
 end_time = end_time = time.time()
print('Completed encoding of item_condition_id and shipping and time taken::', 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( end_time - start_time))))))<categorify>
columns = [col for col in train_features.columns if col != 'id']
woe_encoder = ce.WOEEncoder(cols=columns)
 woe_encoded_train = woe_encoder.fit_transform(train_features[columns], train_targets ).add_suffix('_woe')
 train_features = train_features.join(woe_encoded_train)
 woe_encoded_cols = woe_encoded_train.columns
start_time = time.time()
print('Starting encoding Brand and newCategory and start time is:' , 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( start_time))))))
le = LabelEncoder()
le.fit(np.hstack([combdata.newCategory]))
combdata['lbl_category'] = le.transform(combdata.newCategory)
 le.fit(np.hstack([combdata.Brand]))
combdata['lbl_brand_name'] = le.transform(combdata.Brand)
 del le
gc.collect()
end_time = time.time()
print('Completed encoding Brand and newCategory and time taken to process:', 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( end_time - start_time))))))<feature_engineering>
df = train_features.copy()
df['target'] = train_targets
overall_number_of_ones = train_targets.sum()
overall_number_of_zeroes = 600000 - overall_number_of_ones
print("There are {} ones and {} zeroes in the training set".format(
 overall_number_of_ones, overall_number_of_zeroes 
))
grouped = pd.DataFrame()
grouped['Total'] = df.groupby('nom_0' ).id.count()
grouped['number of ones'] = df.groupby('nom_0' ).target.sum()
grouped['number of zeroes'] = grouped['Total'] - grouped['number of ones']
grouped['percentage of ones'] = grouped['number of ones'] / overall_number_of_ones
grouped['percentage of zeroes'] = grouped['number of zeroes'] / overall_number_of_zeroes
grouped['(% ones)>(% zeroes)'] = grouped['percentage of ones'] > grouped['percentage of zeroes']
grouped['weight of evidence'] = df.groupby('nom_0' ).nom_0_woe.mean()
grouped
def buildWordVector(tokens, size, vec):
count = 0.
 for words in tokens.split(' '):
try:
vec = vec + word2vecModel.wv.get_vector(words ).reshape(( 1, size)) * tfidf[words]
count += 1.
 except KeyError:
continue
if count != 0:
vec /= count
return vec<split>
grouped = pd.DataFrame()
grouped['Total'] = df.groupby('month' ).id.count()
grouped['number of ones'] = df.groupby('month' ).target.sum()
grouped['number of zeroes'] = grouped['Total'] - grouped['number of ones']
grouped['percentage of ones'] = grouped['number of ones'] / overall_number_of_ones
grouped['percentage of zeroes'] = grouped['number of zeroes'] / overall_number_of_zeroes
grouped['(% ones)>(% zeroes)'] = grouped['percentage of ones'] > grouped['percentage of zeroes']
grouped['weight of evidence'] = df.groupby('month' ).month_woe.mean()
grouped
start_time = time.time()
print('Spliting train and test data and start time is:' , 
(time.strftime("%Hhrs:%Mm:%Ss", time.gmtime(( start_time)))))
 print('The original train no of rows(%d)' %(nrow_train))
org_train = combdata[:nrow_train]
X_test = combdata[nrow_train:]
print(' org_train shape ', org_train.shape)
 print(' X_test shape ', X_test.shape)
 print(' Spliting training data into train and validation set...')
 X_splt_train, X_splt_val = train_test_split(org_train, random_state=42, test_size=0.10)
 y_train = X_splt_train['price'].values
y_train = y_train.reshape(len(y_train),1)
 y_val = X_splt_val['price'].values
y_val = y_val.reshape(len(y_val),1)
 X_TRAIN_SHAPE = X_splt_train.shape[1]
Y_TRAIN_SHAPE = y_train.shape[1]
X_VAL_SHAPE = X_splt_val.shape[1]
Y_VAL_SHAPE = y_val.shape[1]
print(' X_splt_train shape ', X_splt_train.shape)
 print(' X_splt_val shape ', X_splt_val.shape)
 print(' y_train shape ', y_train.shape)
 print(' y_val shape ', y_val.shape)
 del combdata
gc.collect()
end_time = time.time()
print('Completed spliting train and test data and time taken::', 
(time.strftime("%Hhrs:%Mm:%Ss",(time.gmtime(( end_time - start_time))))))<prepare_x_and_y>
def logreg_test(cols, encoder):
df = train_features[cols]
auc_scores = []
acc_scores = []
skf = StratifiedKFold(n_splits=6, shuffle=True ).split(df, train_targets)
 for train_id, valid_id in skf:
enc_tr = encoder.fit_transform(df.iloc[train_id,:], train_targets.iloc[train_id])
 enc_val = encoder.transform(df.iloc[valid_id,:])
 regressor = LogisticRegression(solver='lbfgs', max_iter=1000, C=0.6)
 regressor.fit(enc_tr, train_targets.iloc[train_id])
 acc_scores.append(regressor.score(enc_val, train_targets.iloc[valid_id]))
probabilities = [pair[1] for pair in regressor.predict_proba(enc_val)]
auc_scores.append(roc_auc_score(train_targets.iloc[valid_id], probabilities))
acc_scores = pd.Series(acc_scores)
 mean_acc = acc_scores.mean() * 100
print("Mean accuracy score: {:.3f}%".format(mean_acc))
auc_scores = pd.Series(auc_scores)
 mean_auc = auc_scores.mean() * 100
print("Mean AUC score: {:.3f}%".format(mean_auc))
print("Using Weight of Evidence Encoder")
 woe_encoder = ce.WOEEncoder(cols=columns)
 logreg_test(columns, woe_encoder)
 print("
Using Target Encoder")
 targ_encoder = ce.TargetEncoder(cols=columns, smoothing=0.2)
 logreg_test(columns, targ_encoder)
 print("
Using CatBoost Encoder")
 cb_encoder = ce.CatBoostEncoder(cols=columns)
 logreg_test(columns, cb_encoder )
def get_tflearn_data(dataset, scaler):
print(' Creating data dictionary...')
 wordVecSize = 200
zeroVec = np.zeros(wordVecSize, dtype=np.float32 ).reshape(( 1, wordVecSize))
print(' Creating name feature...')
 name = np.concatenate([buildWordVector(z, wordVecSize, zeroVec)for z in dataset.newName], axis=0)
 print(' The shape of namefeature...', name.shape)
 print(' Creating description feature...')
 description = np.concatenate([buildWordVector(z, wordVecSize, zeroVec)for z in dataset.description], axis=0)
 print(' The shape of description feature...', description.shape)
 brand_name = np.array(dataset.lbl_brand_name.values)
 brand_name = brand_name.reshape(len(brand_name),1)
 category = np.array(dataset.lbl_category.values)
 category = category.reshape(len(category),1)
 hot_shp_item_cnd = np.array(dataset.item_condition_id.values)
 item_condition = np.array(dataset.item_condition_id.values)
 item_condition = item_condition.reshape(len(item_condition), 1)
 shipping = np.array(dataset.shipping.values)
 shipping = shipping.reshape(len(shipping), 1)
 print(' Scaling inputs features...')
 name = scaler.fit_transform(name)
 description = scaler.fit_transform(description)
 brand_name = scaler.fit_transform(brand_name)
 category = scaler.fit_transform(category)
 item_condition = scaler.fit_transform(item_condition)
 shipping = scaler.fit_transform(shipping)
 print(' Finished scaling inputs features...')
 X = {
'name': name,
'item_desc' : description,
'brand_name' : brand_name,
'category' : category,
'item_condition' : item_condition,
'shipping' : shipping
}
print(' Finished data dictionary...')
 return(X)
<create_dataframe>
encoder = ce.TargetEncoder(cols=columns, smoothing=0.2)
 encoded_train = encoder.fit_transform(train_features[columns], train_targets ).add_suffix('_targ_enc')
 train_features = train_features.join(encoded_train)
 encoder = ce.CatBoostEncoder(cols=columns)
 encoded_train = encoder.fit_transform(train_features[columns], train_targets ).add_suffix('_catboost')
 train_features = train_features.join(encoded_train)
 training_set = train_features.copy()
training_set['target'] = train_targets
corrmat = training_set.corr()
plt.subplots(figsize=(20,20))
sns.heatmap(corrmat, vmax=0.9, square=True )
<prepare_x_and_y><EOS> 
df = train_features[columns]
train_encoded = pd.DataFrame()
skf = StratifiedKFold(n_splits=5,shuffle=True ).split(df, train_targets)
 for tr_in,fold_in in skf:
encoder = ce.WOEEncoder(cols=columns)
 encoder.fit(df.iloc[tr_in,:], train_targets.iloc[tr_in])
 train_encoded = train_encoded.append(encoder.transform(df.iloc[fold_in,:]),ignore_index=False)
 train_encoded = train_encoded.sort_index()
encoder = ce.WOEEncoder(cols=columns)
 encoder.fit(df, train_targets)
 test_encoded = encoder.transform(test_features[columns])
 regressor = LogisticRegression(solver='lbfgs', max_iter=1000, C=0.6)
 regressor.fit(train_encoded, train_targets)
 probabilities = [pair[1] for pair in regressor.predict_proba(test_encoded)]
output = pd.DataFrame({'id': test_features['id'],
'target': probabilities})
 output.to_csv('submission.csv', index=False)
 output.describe()