Kaggle competition: Jane Street Market Prediction
df_melted['test'] = df_melted[["id","sales"]].groupby("id")["sales"].shift(1 ).fillna(-1 ).astype(np.int16)
 print(list(df_melted[df_melted['day']==1101]['test'])== list(df_melted[df_melted['day']==1100]['sales']))<drop_column>
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('weight>0' ).reset_index(drop=True)
 train.fillna(train.median() ,inplace=True)
 train['feature_stock_id_sum'] = train['feature_41'] + train['feature_42'] + train['feature_43']
train['feature_1_2_cross'] = train['feature_1']/(train['feature_2']+1e-5)
 NUM_TRAIN_EXAMPLES = len(train )
df_melted.drop(columns=['test'], inplace=True )<categorify>
features = [c for c in train.columns if 'feature' in c]
f_mean = np.nanmean(train[features[1:]].values,axis=0 )
for lag in tqdm([28, 35, 42, 49, 56, 63, 70, 77]):
df_melted["lag_" + str(lag)] = df_melted[["id","sales"]].groupby("id")["sales"].shift(lag ).fillna(-1 ).astype(np.int16 )<filter>
PATH ='.. /input/neutralizing2'
df_melted=df_melted[df_melted['lag_77']!=-1]<filter>
resp_cols = [c for c in train.columns if 'resp' in c]
assert list(df_melted[df_melted['day']==1528]['lag_28'])== list(df_melted[df_melted['day']==1500]['sales'] )<merge>
def mish(x):
return tf.keras.layers.Lambda(lambda x: x*K.tanh(K.softplus(x)) )(x)
 tf.keras.utils.get_custom_objects().update({'mish': tf.keras.layers.Activation(mish)})
 def create_model(input_shape):
inp = tf.keras.layers.Input(input_shape)
 tmp = tf.keras.layers.BatchNormalization()(inp)
 xs = [tmp]
for _ in range(10):
if len(xs)> 1:
tmp = tf.keras.layers.Concatenate(axis=-1 )(xs)
 else:
tmp = xs[0]
tmp = tf.keras.layers.Dense(64,activation='mish' )(tmp)
 tmp = tf.keras.layers.BatchNormalization()(tmp)
 tmp = tf.keras.layers.Dropout(0.2 )(tmp)
 xs.append(tmp)
 output = tf.keras.layers.Dense(len(resp_cols),activation='sigmoid' )(tf.keras.layers.Concatenate()(xs))
model = tf.keras.models.Model(inp,output)
 optimizer = tfa.optimizers.RectifiedAdam(1e-3)
 model.compile(optimizer, loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.001),
metrics=[tf.keras.metrics.BinaryAccuracy(name='binary_accuracy')])
 return model
df_melted=df_melted.merge(cal_df[['d', 'wm_yr_wk']], left_on=['day'], right_on=['d'] ).drop(columns=['d'] )<merge>
def set_all_seeds(seed):
np.random.seed(seed)
 random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed )
df_melted=df_melted.merge(price_df[['id', 'sell_price', 'wm_yr_wk']], on=['id', 'wm_yr_wk'], how='inner' )<drop_column>
class NeutralizeTransform:
def __init__(self,proportion=1.0):
self.proportion = proportion
def fit(self,X,y):
self.lms = []
self.mean_exposure = np.mean(y,axis=0)
 self.y_shape = y.shape[-1]
for x in X.T:
scores = x.reshape(( -1,1))
exposures = y
exposures = np.hstack(( exposures, np.array([np.mean(scores)] * len(exposures)).reshape(-1, 1)))
 transform = np.linalg.lstsq(exposures, scores, rcond=None)[0]
self.lms.append(transform)
 def transform(self,X,y=None):
out = []
for i,transform in enumerate(self.lms):
x = X[:,i]
scores = x.reshape(( -1,1))
exposures = np.repeat(self.mean_exposure,len(x),axis=0 ).reshape(( -1,self.y_shape))
exposures = np.concatenate([exposures,np.array([np.mean(scores)] * len(exposures)).reshape(( -1,1)) ],axis=1)
 correction = self.proportion * exposures.dot(transform)
 out.append(x - correction.ravel())
 return np.asarray(out ).T
def fit_transform(self,X,y):
self.fit(X,y)
 return self.transform(X,y)
del price_df
del cal_df<import_modules>
TRAINING = False
import lightgbm as lgb<init_hyperparams>
%%time
if TRAINING:
mask = train[features].isna()
train.fillna(0,inplace=True)
 for feature in features:
nt = NeutralizeTransform(proportion=0.25)
 train[feature] = nt.fit_transform(train[feature].values.reshape(( -1,1)) ,
train['resp'].values.reshape(( -1,1)))
 pd.to_pickle(nt,f'NeutralizeTransform_{feature}.pkl')
 train[mask] = np.nan
else:
nts = []
for feature in features:
nt = pd.read_pickle(f'{PATH}/NeutralizeTransform_{feature}.pkl')
 nts.append(nt )
best_params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.05,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : 2048,
'num_leaves': 64,
"min_data_in_leaf": 50,
}<drop_column>
gc.collect()
del wk_price_df<prepare_x_and_y>
TRAINING = False
X_train = df_melted[df_melted["day"] < 1886].drop(columns=["sales"])
 X_val = df_melted[df_melted["day"].between(1886, 1913)].drop(columns=["sales"])
 X_test = df_melted[df_melted["day"] > 1913].drop(columns=["sales"])
 y_train = df_melted[df_melted["day"] < 1886]["sales"]
y_val = df_melted[df_melted["day"].between(1886, 1913)]["sales"]<create_dataframe>
X_tr = train.query('date<350')[features].values
y_tr =(train.query('date<350')[resp_cols].values > 0 ).astype(int)
 X_val = train.query('date>400')[features].values
y_val =(train.query('date>400')[resp_cols].values > 0 ).astype(int)
 del train
gc.collect()
if TRAINING:
metric = {}
for seed in [2020,1982]:
set_all_seeds(seed)
 model = create_model(X_tr.shape[-1])
 hist = model.fit(X_tr,y_tr,
validation_data=(X_val,y_val),
epochs=200,
batch_size=8192,
callbacks=[tf.keras.callbacks.EarlyStopping('val_binary_accuracy',mode='max',patience=20,restore_best_weights=True),
tf.keras.callbacks.ReduceLROnPlateau('val_binary_accuracy',mode='max',patience=10,cooldown=5)])
 model.save_weights(f'model_{seed}.tf')
 metric[seed] = max(hist.history['val_binary_accuracy'])
 tf.keras.backend.clear_session()
print(metric)
 else:
models = []
for seed in [2020,1982]:
model = create_model(X_tr.shape[-1])
 model.load_weights(f'{PATH}/model_{seed}.tf')
 model.call = tf.function(model.call, experimental_relax_shapes=True)
 models.append(model)
<drop_column><EOS> 
if not TRAINING:
f = np.median
env = janestreet.make_env()
th = 0.495
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].values[0] > 0:
test_df['feature_stock_id_sum'] = test_df['feature_41'] + test_df['feature_42'] + test_df['feature_43']
test_df['feature_1_2_cross'] = test_df['feature_1']/(test_df['feature_2']+1e-5)
 x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
for i in range(len(nts)) :
x_tt[:,i] = nts[i].transform(np.expand_dims(x_tt[:,i],0))
p = f(np.mean([model(x_tt,training=False ).numpy() for model in models],axis=0))
pred_df.action = np.where(p > th, 1, 0 ).astype(int)
 else:
pred_df["action"].values[0] = 0
env.predict(pred_df )