Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<drop_column>
warnings.filterwarnings("ignore")
def get_w(news_train_df):
w = news_train_df
w['assetCode'] = news_train_df['assetCodes']
try:
w.drop(columns=['assetCodes'], inplace=True)
 except:
pass
try:
w.drop(columns=['assetName'], inplace=True)
 except:
pass
try:
w.drop(columns=['sourceTimestamp'], inplace=True)
 except:
pass
try:
w.drop(columns=['firstCreated'], inplace=True)
 except:
pass
try:
w.drop(columns=['sourceId'], inplace=True)
 except:
pass
try:
w.drop(columns=['headline'], inplace=True)
 except:
pass
try:
w.drop(columns=['takeSequence'], inplace=True)
 except:
pass
try:
w.drop(columns=['provider'], inplace=True)
 except:
pass
try:
w.drop(columns=['subjects'], inplace=True)
 except:
pass
try:
w.drop(columns=['audiences'], inplace=True)
 except:
pass
try:
w.drop(columns=['bodySize'], inplace=True)
 except:
pass
try:
w.drop(columns=['companyCount'], inplace=True)
 except:
pass
try:
w.drop(columns=['headlineTag'], inplace=True)
 except:
pass
try:
w.drop(columns=['marketCommentary'], inplace=True)
 except:
pass
try:
w.drop(columns=['sentenceCount'], inplace=True)
 except:
pass
try:
w.drop(columns=['firstMentionSentence'], inplace=True)
 except:
pass
try:
w.drop(columns=['urgency'], inplace=True)
 except:
pass
try:
w.drop(columns=['relevancy'], inplace=True)
 except:
pass
try:
w.drop(columns=['sentimentClass'], inplace=True)
 except:
pass
try:
w.drop(columns=['noveltyCount12H'], inplace=True)
 except:
pass
try:
w.drop(columns=['noveltyCount24H'], inplace=True)
 except:
pass
try:
w.drop(columns=['noveltyCount3D'], inplace=True)
 except:
pass
try:
w.drop(columns=['noveltyCount5D'], inplace=True)
 except:
pass
try:
w.drop(columns=['noveltyCount7D'], inplace=True)
 except:
pass
try:
w.drop(columns=['volumeCounts12H'], inplace=True)
 except:
pass
try:
w.drop(columns=['volumeCounts24H'], inplace=True)
 except:
pass
try:
w.drop(columns=['volumeCounts3D'], inplace=True)
 except:
pass
try:
w.drop(columns=['volumeCounts5D'], inplace=True)
 except:
pass
try:
w.drop(columns=['volumeCounts7D'], inplace=True)
 except:
pass
w['ratio'] = news_train_df['sentimentWordCount']/news_train_df['wordCount']
try:
w.drop(columns=['sentimentWordCount'], inplace=True)
 except:
pass
try:
w.drop(columns=['wordCount'], inplace=True)
 except:
pass
try:
w.drop(columns=['sentimentNegative'], inplace=True)
 except:
pass
try:
w.drop(columns=['sentimentNeutral'], inplace=True)
 except:
pass
return w<predict_on_test>
def auc(y_true, y_pred):
def fallback_auc(y_true, y_pred):
try:
return metrics.roc_auc_score(y_true, y_pred)
 except:
return 0.5
return tf.py_function(fallback_auc,(y_true, y_pred), tf.double )
def get_xwy(market_train_df, news_train_df):
y = get_y(market_train_df)
 x = get_x(market_train_df)
 w = get_w(news_train_df)
 return x, w, y		
def make_predictions(predictions_template_df, market_obs_df, news_obs_df):
			
x = get_x(market_obs_df)
 w = get_w(news_obs_df)
 cdf = pd.merge(x, w, how='left', on=['time', 'assetCode'], copy=False)
 cdf = cdf[train_cols]
data = xgb.DMatrix(cdf, nthread=-1)
 predictions_template_df.confidenceValue = np.clip(m.predict(data,ntree_limit=0),-1,1)
<prepare_x_and_y>
def create_model(data, catcols):
inputs = []
outputs = []
for c in catcols:
num_unique_values = int(data[c].nunique())
 embed_dim = int(min(np.ceil(( num_unique_values)/2), 50))
inp = layers.Input(shape=(1,))
out = layers.Embedding(num_unique_values + 1, embed_dim, name=c )(inp)
 out = layers.SpatialDropout1D(0.3 )(out)
 out = layers.Reshape(target_shape=(embed_dim,))(out)
 inputs.append(inp)
 outputs.append(out)
 x = layers.Concatenate()(outputs)
 x = layers.BatchNormalization()(x)
 x = layers.Dense(300, activation="relu" )(x)
 x = layers.Dropout(0.3 )(x)
 x = layers.BatchNormalization()(x)
 x = layers.Dense(300, activation="relu" )(x)
 x = layers.Dropout(0.3 )(x)
 x = layers.BatchNormalization()(x)
 y = layers.Dense(2, activation="softmax" )(x)
 model = Model(inputs=inputs, outputs=y)
 return model
( market_train_df, news_train_df)= env.get_training_data()
print('Datos y News Recuperados!')
 x, w, y = get_xwy(market_train_df, news_train_df)
 del market_train_df, news_train_df
print('Get XWY!')
 cdf = pd.merge(x, w, how='left', on=['time', 'assetCode'], copy=False)
 print(w['time'])
 print(w['assetCode'])
 print(format(cdf['time'].isnull().sum()))
print('Merge!')
 w.drop(columns=['time'], inplace=True)
 x.drop(columns=['time'], inplace=True)
 cdf.drop(columns=['time'], inplace=True)
 w.drop(columns=['assetCode'], inplace=True)
 x.drop(columns=['assetCode'], inplace=True)
 cdf.drop(columns=['assetCode'], inplace=True)
 data = xgb.DMatrix(cdf, label=y, feature_names=cdf.columns)
 print('Get Matrix!')
 train_cols = cdf.columns
params = {'max_depth':10,'min_child_weight': 1,'eta':.8,'objective':'reg:linear','eval_metric:' : "mae",}
m = xgb.train(params, data, 3, evals=[(data, 'Test')])
 print('Train!')
 for(market_obs_df, news_obs_df, predictions_template_df)in env.get_prediction_days() :
make_predictions(predictions_template_df, market_obs_df, news_obs_df)
 env.predict(predictions_template_df)
 break
print('Ok!')
 xgb.plot_importance(m)
 env.write_submission_file() 
<set_options>
train = pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv")
 test = pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv")
 sample = pd.read_csv("/kaggle/input/cat-in-the-dat/sample_submission.csv" )
pd.set_option('max_columns', 50 )<split>
test["target"] = -1
data = pd.concat([train, test] ).reset_index(drop=True )
env = twosigmanews.make_env()
market_train_df, news_train = env.get_training_data()<data_type_conversions>
features = [x for x in train.columns if x not in ["id", "target"]]
for column in market_train_df.select_dtypes(include='float64' ).columns:
market_train_df[column] = market_train_df[column].astype('float16' )<feature_engineering>
for feat in features:
lbl_enc = preprocessing.LabelEncoder()
data[feat] = lbl_enc.fit_transform(data[feat].values )
start = datetime(2010, 1, 1, 0, 0, 0 ).date()
market_train = market_train_df.loc[market_train_df['time'].dt.date >= start].reset_index(drop=True)
 news_train = news_train.loc[news_train['time'].dt.date >= start].reset_index(drop=True )<drop_column>
train = data[data.target != -1].reset_index(drop=True)
 test = data[data.target == -1].reset_index(drop=True)
 test_data = [test.loc[:, features].values[:, k] for k in range(test.loc[:, features].values.shape[1])]
def preprocess_news(news_train):
drop_list = [
'audiences', 'subjects', 'assetName',
'headline', 'firstCreated', 'sourceTimestamp',
]
news_train.drop(drop_list, axis=1, inplace=True)
 for col in ['headlineTag', 'provider', 'sourceId']:
news_train[col], uniques = pd.factorize(news_train[col])
 del uniques
news_train['assetCodes'] = news_train['assetCodes'].apply(lambda x: x[1:-1].replace("'", ""))
return news_train<string_transform>
oof_preds = np.zeros(( len(train)))
 test_preds = np.zeros(( len(test)))
 skf = StratifiedKFold(n_splits=50)
 for train_index, test_index in skf.split(train, train.target.values):
X_train, X_test = train.iloc[train_index, :], train.iloc[test_index, :]
X_train = X_train.reset_index(drop=True)
 X_test = X_test.reset_index(drop=True)
 y_train, y_test = X_train.target.values, X_test.target.values
model = create_model(data, features)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[auc])
 X_train = [X_train.loc[:, features].values[:, k] for k in range(X_train.loc[:, features].values.shape[1])]
X_test = [X_test.loc[:, features].values[:, k] for k in range(X_test.loc[:, features].values.shape[1])]
es = callbacks.EarlyStopping(monitor='val_auc', min_delta=0.001, patience=5,
verbose=1, mode='max', baseline=None, restore_best_weights=True)
 rlr = callbacks.ReduceLROnPlateau(monitor='val_auc', factor=0.5,
patience=3, min_lr=1e-6, mode='max', verbose=1)
 model.fit(X_train,
utils.to_categorical(y_train),
validation_data=(X_test, utils.to_categorical(y_test)) ,
verbose=1,
batch_size=1024,
callbacks=[es, rlr],
epochs=100 
)
 valid_fold_preds = model.predict(X_test)[:, 1]
test_fold_preds = model.predict(test_data)[:, 1]
oof_preds[test_index] = valid_fold_preds.ravel()
test_preds += test_fold_preds.ravel()
print(metrics.roc_auc_score(y_test, valid_fold_preds))
K.clear_session()
news_train = preprocess_news(news_train )<drop_column>
print("Overall AUC={}".format(metrics.roc_auc_score(train.target.values, oof_preds)) )
<drop_column><EOS> 
test_preds /= 50
test_ids = test.id.values
print("Saving submission file")
 submission = pd.DataFrame.from_dict({
'id': test_ids,
'target': test_preds
})
 submission.to_csv("submission.csv", index=False )