Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<categorify>
pd.set_option('display.max_columns', 500)
 warnings.filterwarnings("ignore" )
df_melted = temp.melt(id_vars=[n for n in temp.columns if n.find("id")!=-1],
value_vars=[n for n in temp.columns if n.find("d_")==0],
var_name = 'day', value_name = 'sales')
 del temp<feature_engineering>
%%time
train_cudf = cudf.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 train = train_cudf.to_pandas()
del train_cudf
features = pd.read_csv('.. /input/jane-street-market-prediction/features.csv')
 example_test = pd.read_csv('.. /input/jane-street-market-prediction/example_test.csv')
 sample_prediction_df = pd.read_csv('.. /input/jane-street-market-prediction/example_sample_submission.csv')
 print("Data is loaded!" )
df_melted["day"]=df_melted["day"].apply(lambda x: int(x.split("_")[1]))<merge>
missing_values_count = train.isnull().sum()
print(missing_values_count)
 total_cells = np.product(train.shape)
 total_missing = missing_values_count.sum()
print("% of missing data = ",(total_missing/total_cells)* 100 )
df_melted=df_melted.merge(cal_df.drop(columns=["date", "wm_yr_wk",
"weekday"]), left_on=["day"], right_on=["d"] ).drop(columns=["d"] )<count_values>
train = train[train['weight'] != 0]
train = train.query('date > 85' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train['action'] =(( train['weight'].values * train['resp'].values)> 0 ).astype('int')
 train.fillna(train.mean() ,inplace=True)
 cols = [c for c in train.columns if 'feature' in c]
X_train = train.loc[:, train.columns.str.contains('feature')]
y_train = train.loc[:, 'action']
df_melted['event_name_1'].value_counts(dropna=False ).head()<data_type_conversions>
del x, y, train, features, example_test, sample_prediction_df
df_melted['event_name_1'].astype('category' ).cat.codes.astype("int8" ).value_counts().head()<data_type_conversions>
print("XGBoost version:", xgb.__version__ )
df_melted["event_name_1"]=df_melted["event_name_1"].astype('category' ).cat.codes.astype("int8")
 df_melted["event_name_2"]=df_melted["event_name_2"].astype('category' ).cat.codes.astype("int8")
 df_melted["event_type_1"]=df_melted["event_type_1"].astype('category' ).cat.codes.astype("int8")
 df_melted["event_type_2"]=df_melted["event_type_2"].astype('category' ).cat.codes.astype("int8" )<data_type_conversions>
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=11,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.7,
missing=-999,
random_state=2020,
tree_method='gpu_hist'
)
useful_ids = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']
id_encodings = [id_col + '_encoding' for id_col in useful_ids]
for id_col in useful_ids:
if id_col == 'item_id':
df_melted[id_col + '_encoding'] = df_melted[id_col].astype('category' ).cat.codes.astype("int16")
 else:
df_melted[id_col + '_encoding'] = df_melted[id_col].astype('category' ).cat.codes.astype("int8" )<drop_column>
%time clf.fit(X_train, y_train )
df_melted.drop(columns=['year'] + useful_ids, inplace=True )<data_type_conversions>
TRAINING = True
start_time = time.time()
if TRAINING:
env = janestreet.make_env()
th = 0.5
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, test_df.columns.str.contains('feature')].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])
 pred = clf.predict(x_tt)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df)
 print(f"took: {time.time() - start_time} seconds" )