Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<data_type_conversions>
warnings.filterwarnings('ignore')
 PATH = '/kaggle/input/cat-in-the-dat/'
def fill_in(data):
for i in data.columns:
if data[i].dtype == "object":
data[i] = data[i].fillna("")
 elif(data[i].dtype == "int64" or data[i].dtype == "float64"):
data[i] = data[i].fillna(data[i].mean())
 else:
pass
return data
market_train_df = fill_in(market_train_df )<categorify>
train = pd.read_csv(f'{PATH}train.csv')
 test = pd.read_csv(f'{PATH}test.csv' )
def data_other(market_train_df):
lbl = {k: v for v, k in enumerate(market_train_df['assetCode'].unique())}
market_train_df['assetCodeT'] = market_train_df['assetCode'].map(lbl)
 market_train_df = market_train_df.dropna(axis=0)
 return market_train_df
market_train_df = data_other(market_train_df )<filter>
%%time
target = train['target']
train_id = train['id']
test_id = test['id']
train.drop(['target', 'id', 'bin_0'], axis=1, inplace=True)
 test.drop(['id','bin_0'], axis=1, inplace=True)
 print(train.shape)
 print(test.shape )
green = market_train_df.returnsOpenNextMktres10 > 0
green = green.values<define_variables>
%%time
traintest = pd.concat([train, test])
 dummies = pd.get_dummies(traintest, columns=traintest.columns, drop_first=True, sparse=True)
 train_ohe = dummies.iloc[:train.shape[0], :]
test_ohe = dummies.iloc[train.shape[0]:, :]
print(train_ohe.shape)
 print(test_ohe.shape )
fcol = [c for c in market_train_df if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', 'universe','sourceTimestamp']]<define_variables>
%%time
train_ohe = train_ohe.sparse.to_coo().tocsr()
test_ohe = test_ohe.sparse.to_coo().tocsr()
X = market_train_df[fcol].values
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng )<define_variables>
print('
print('StratifiedKFold training...')
 folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=10)
 model = LogisticRegression(solver='lbfgs', max_iter=200, C=0.095)
 score = []
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_ohe, target, groups=target)) :
print('Fold:',fold_+1)
 tr_x, tr_y = train_ohe[trn_idx,:], target[trn_idx]
vl_x, v_y = train_ohe[val_idx,:], target[val_idx]
%time model.fit(tr_x, tr_y)
 y_pred = model.predict(vl_x)
 score.append(roc_auc_score(v_y, y_pred))
print(f'AUC score : {roc_auc_score(v_y, y_pred)}')
 print('Average AUC score', np.mean(score))
print('
all_10day = market_train_df.returnsOpenNextMktres10.values<split>
%time model.fit(train_ohe, target )
X_train, X_test, green_train, green_test, all_train, all_test = model_selection.train_test_split(
 X, green, all_10day, test_size=0.20, random_state=59 )<data_type_conversions>
pred_test = model.predict_proba(test_ohe)[:,0]
<prepare_x_and_y><EOS> 
sub = pd.read_csv(f'{PATH}sample_submission.csv')
 sub['target'] = pred_test
sub.to_csv('log_reg_0.1.csv', index=None, header=True )