Kaggle competition: Categorical Feature Encoding Challenge
del market_train_orig
del news_train_orig<merge>
train=pd.read_csv('/kaggle/input/cat-in-the-dat/train.csv', index_col='id')
 test=pd.read_csv('/kaggle/input/cat-in-the-dat/test.csv', index_col='id')
 submission=pd.read_csv('/kaggle/input/cat-in-the-dat/sample_submission.csv', index_col='id')
 Ytrain=train['target']
train=train[list(test)]
all_data=pd.concat(( train, test))
print(train.shape, test.shape, all_data.shape )
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
n_lag = [3,7,14]
new_df = generate_lag_features(market_train_df,n_lag=n_lag)
 market_train_df = pd.merge(market_train_df,new_df,how='left',on=['time','assetCode'] )<set_options>
encoded=pd.get_dummies(all_data, columns=all_data.columns, sparse=True)
 encoded=encoded.sparse.to_coo()
encoded=encoded.tocsr()
del new_df
gc.collect()<feature_engineering>
Xtrain=encoded[:len(train)]
Xtest=encoded[len(train):]
def prepare_news_data(news_df):
news_df['position'] = news_df['firstMentionSentence'] / news_df['sentenceCount']
news_df['coverage'] = news_df['sentimentWordCount'] / news_df['wordCount']
droplist = ['sourceTimestamp','firstCreated','sourceId','headline','takeSequence','provider','firstMentionSentence',
'sentenceCount','bodySize','headlineTag','marketCommentary','subjects','audiences','sentimentClass',
'assetName', 'urgency','wordCount','noveltyCount12H','sentimentWordCount','noveltyCount24H','noveltyCount3D','noveltyCount5D','noveltyCount7D','volumeCounts12H','volumeCounts24H','volumeCounts3D','volumeCounts5D','volumeCounts7D',]
news_df.drop(droplist, axis=1, inplace=True)
 assets = []
indices = []
for i, values in news_df['assetCodes'].iteritems() :
assetCodes = eval(values)
 assets.extend(assetCodes)
 indices.extend([i]*len(assetCodes))
mapping_df = pd.DataFrame({'news_index': indices, 'assetCode': assets})
 del assets, indices
news_df['news_index'] = news_df.index.copy()
expanded_news_df = mapping_df.merge(news_df, how='left', on='news_index')
 del mapping_df, news_df
expanded_news_df.drop(['news_index', 'assetCodes'], axis=1, inplace=True)
 return expanded_news_df.groupby(['time', 'assetCode'] ).mean().reset_index()<drop_column>
kf=StratifiedKFold(n_splits=10 )
news_train_df = prepare_news_data(news_train_df )<merge>
def objective(trial):
C=trial.suggest_loguniform('C', 10e-10, 10)
 model=LogisticRegression(C=C, class_weight='balanced',max_iter=10000, solver='lbfgs', n_jobs=-1)
 score=-cross_val_score(model, Xtrain, Ytrain, cv=kf, scoring='roc_auc' ).mean()
return score
study=optuna.create_study()
<set_options><EOS> 
model=LogisticRegression(C=0.09536298444122952, class_weight='balanced',max_iter=10000, solver='lbfgs', n_jobs=-1)
 model.fit(Xtrain, Ytrain)
 predictions=model.predict_proba(Xtest)[:,1]
submission['target']=predictions
submission.to_csv('Result.csv')
 submission.head()