Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<train_model>
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score,classification_report
import seaborn as sns
import pandas as pd
import numpy as np
env = twosigmanews.make_env() 
(market_train_df, news_train_df)= env.get_training_data()<define_variables>
submission = pd.read_csv("/kaggle/input/cat-in-the-dat/sample_submission.csv")
 train = pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv")
 test = pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv")
 labels = train['target'].values
train_id = train.pop("id")
 test_id = test.pop("id" )
print(market_train_df.time.min())
 market_train_df_5years=market_train_df[market_train_df.time>'2011-12-30 22:00:00+0000'] 
<save_to_csv>
data = pd.concat([train.drop('target',axis=1), test])
 totaal=train.append(test )
market_train_df_5years.to_csv("market_2011_2016.csv",index=False )<set_options>
cols=[ci for ci in train.columns if ci not in ['id','index','target']]
coltype=train.dtypes
for ci in cols:
if(coltype[ci]=="object"):
codes=totaal[[ci,'target']].groupby(ci ).mean().sort_values("target")
 codesdict=codes.target.to_dict()
totaal[ci]=totaal[ci].map(codesdict)
%matplotlib inlinemarket_train_df.info()
py.init_notebook_mode(connected=True)
<data_type_conversions>
totaal['id']=train_id.append(test_id)
 totaal=totaal.fillna(0 )
missing_cols=['returnsClosePrevMktres1',
'returnsOpenPrevMktres1',
'returnsClosePrevMktres10',
'returnsOpenPrevMktres10']
for col in missing_cols:
market_train_df[col]=market_train_df[col].fillna(market_train_df[col].mean() )<train_model>
columns = [i for i in test.columns]
dummies = pd.get_dummies(data,
columns=columns,
drop_first=True,
sparse=True,dtype='float')
import datetimeprint(f'The market data start from {market_train_df["time"].min() }, and end on {market_train_df["time"].max() }' )<define_variables>
sparse_matrix = lil_matrix(( len(totaal), len(totaal.columns)))
 for k, column_name in enumerate(data.columns):
sparse_matrix[totaal.id.values, np.full(len(totaal), k)] = totaal[column_name].values
return_terms=[
"returnsClosePrevRaw1",
"returnsOpenPrevRaw1",
"returnsClosePrevMktres1",
"returnsOpenPrevMktres1",
"returnsClosePrevRaw10",
"returnsOpenPrevRaw10",
"returnsClosePrevMktres10",
"returnsOpenPrevMktres10",
"returnsOpenNextMktres10"
]
for return_term in return_terms:
market_train_df[return_term]=market_train_df[return_term].clip(-1,1)
<prepare_output>
train = totdum.tocsc() [:train.shape[0], :]
test = totdum.tocsc() [train.shape[0]:, :]
del dummies,totdum,sparse_matrix,totaal,data
print(f'The news data start from {news_train_df["time"].min() }, and end on {news_train_df["time"].max() }')
<feature_engineering>
train = train.astype("float32")
 test = test.astype("float32" )
def expand(df, expand_column):
lens = []
d = {}
expands_list=[]
for i,item in df[expand_column].items() :
expand=list(item[2:-2].split("', '"))
lens.append(len(expand))
expands_list.extend(expand)
 d[expand_column] = expands_list
for col in df.columns.values:
if col != expand_column:
d[col] = np.repeat(df[col].values, lens)
 return pd.DataFrame(d)
<data_type_conversions>
lr = LogisticRegression(solver="lbfgs",max_iter=500,n_jobs=4)
 lengte=100000
lr.fit(train[:lengte], labels[:lengte])
 lr_pred = lr.predict_proba(train[:lengte])[:, 1]
score = roc_auc_score(labels[:lengte], lr_pred)
 print("score: ", score)
 lr_pred = lr.predict(train[:lengte])
 score = classification_report(labels[:lengte], lr_pred)
 print("score: ", score)
 print("First Random float number: ", random.random())
 lr_pred = [(random.random() >0.7)*1 for x in range(lengte)]
score = classification_report(labels[:lengte], lr_pred)
 print("score: ", score)
def prep_market(market_df):
market_df=market_df.drop(['assetName'],axis=1)
 market_df["time"]=pd.to_datetime(market_df["time"])
 market_df['time'] = market_df.time.dt.date
market_df['close_to_open'] = market_df['close'] / market_df['open']
return market_df
def prep_news(news_df):
news_df['firstCreated']=pd.to_datetime(news_df['firstCreated'])
 news_df=news_df.drop(news_drop_col,axis=1)
 news_df['headlineTagT'] = news_df['headlineTag'].map(headlineTag_dic)
 news_df['provider'] = news_df['provider'].map(provider_dic)
 news_df['marketCommentary'] = news_df['marketCommentary'].map(marketCommentary_map)
 news_df=expand(news_df, "assetCodes")
 return news_df
def group_news(news_df):
news_df['firstCreated']=news_df['firstCreated'].dt.date
news_df = news_df.groupby(['firstCreated', 'assetCodes'], as_index=False ).mean()
return news_df
def merge_market_news(market_df,news_df):
market_and_news_df = pd.merge(market_df, news_df, left_on=['time', 'assetCode'], right_on=['firstCreated', 'assetCodes'])
 return market_and_news_df<merge>
submission["id"] = test_id
submission["target"] = lr.predict_proba(test)[:, 1]
market_train_df=prep_market(market_train_df)
 market_news_train_df = merge_market_news(market_train_df, news_train_df)
<define_variables>
submission.to_csv("submission.csv", index=False )
<define_variables><EOS> 
submission.to_csv("submission.csv", index=False )