Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<data_type_conversions>
sns.set_context("talk")
 style.use('fivethirtyeight')
 pd.options.display.max_columns = None
warnings.filterwarnings('ignore' )
def prep_calendar(df):
df = df.drop(["date", "weekday", "event_type_1", "event_type_2"], axis=1)
 df = df.assign(d = df.d.str[2:].astype(int))
to_ordinal = ["event_name_1", "event_name_2"]
df[to_ordinal] = df[to_ordinal].fillna("1")
 df[to_ordinal] = OrdinalEncoder(dtype="int" ).fit_transform(df[to_ordinal])+ 1
to_int8 = ["wday", "month", "snap_CA", "snap_TX", "snap_WI"] + to_ordinal
df[to_int8] = df[to_int8].astype("int8")
 return df
calendar = prep_calendar(calendar)
 calendar.head()<categorify>
print("Tensorflow version " + tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE
MIXED_PRECISION = False
XLA_ACCELERATE = True
if MIXED_PRECISION:
if tpu: policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')
 else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')
 mixed_precision.set_policy(policy)
 print('Mixed precision enabled')
 if XLA_ACCELERATE:
tf.config.optimizer.set_jit(True)
 print('Accelerated Linear Algebra enabled')
LAGS = [7, 28]
WINDOWS = [7, 28, 56]
FIRST = 1914
LENGTH = 28
def demand_features(df):

 for lag in LAGS:
df[f'lag_t{lag}'] = df.groupby('id')['demand'].transform(lambda x: x.shift(lag)).astype("float32")
 for w in WINDOWS:
df[f'rolling_mean_lag{lag}_w{w}'] = df.groupby('id')[f'lag_t{lag}'].transform(lambda x: x.rolling(w ).mean() ).astype("float32")
 return df
def demand_features_eval(df):

 out = df.groupby('id', sort=False ).last()
for lag in LAGS:
out[f'lag_t{lag}'] = df.groupby('id', sort=False)['demand'].nth(-lag-1 ).astype("float32")
 for w in WINDOWS:
out[f'rolling_mean_lag{lag}_w{w}'] = df.groupby('id', sort=False)['demand'].nth(list(range(-lag-w, -lag)) ).groupby('id', sort=False ).mean().astype("float32")
 return out.reset_index()
def prep_data(df, drop_d=1000, dept_id="FOODS_1"):

 print(f"
Working on dept {dept_id}")
 df = df[df.dept_id == dept_id]
df = df.drop(["dept_id", "cat_id"], axis=1)
 df = df.drop(["d_" + str(i+1)for i in range(drop_d)], axis=1)
 df = df.assign(id=df.id.str.replace("_validation", ""))
df = df.reindex(columns=df.columns.tolist() + ["d_" + str(FIRST + i)for i in range(2 * LENGTH)])
 df = df.melt(id_vars=["id", "item_id", "store_id", "state_id"], var_name='d', value_name='demand')
 df = df.assign(d=df.d.str[2:].astype("int64"),
demand=df.demand.astype("float32"))
df = demand_features(df)
 df = df[df.d >(drop_d + max(LAGS)+ max(WINDOWS)) ]
df = df.merge(calendar, how="left", on="d")
 df = df.merge(selling_prices, how="left", on=["store_id", "item_id", "wm_yr_wk"])
 df = df.drop(["wm_yr_wk"], axis=1)
 for v in ["item_id", "store_id", "state_id"]:
df[v] = OrdinalEncoder(dtype="int" ).fit_transform(df[[v]] ).astype("int16")+ 1
x = list(set(df.columns)- {'id', 'd', 'demand'})
 test = df[df.d >= FIRST - max(LAGS)- max(WINDOWS)]
df = df[df.d < FIRST]
xtrain, xvalid, ytrain, yvalid = train_test_split(df[x], df["demand"], test_size=0.1, shuffle=True, random_state=54)
 train = lgb.Dataset(xtrain, label = ytrain)
 valid = lgb.Dataset(xvalid, label = yvalid)
 return train, valid, test, x
def fit_model(train, valid, dept):

 params = {
'metric': 'rmse',
'objective': 'poisson',
'seed': 200,
'learning_rate': 0.2 - 0.13 *(dept in ["HOBBIES_1", "HOBBIES_2", "HOUSEHOLD_2"]),
'lambda': 0.1,
'num_leaves': 50,
'colsample_bytree': 0.7
}
fit = lgb.train(params,
train,
num_boost_round = 5000,
valid_sets = [valid],
early_stopping_rounds = 200,
verbose_eval = 200)
 lgb.plot_importance(fit, importance_type="gain", precision=0, height=0.5, figsize=(6, 10), title=dept);
return fit
def pred_to_csv(fit, test, x, cols=sample_submission.columns, file="submission.csv", first=False):

 for i, day in enumerate(np.arange(FIRST, FIRST + LENGTH)) :
test_day = demand_features_eval(test[(test.d <= day)&(test.d >= day - max(LAGS)- max(WINDOWS)) ])
 test.loc[test.d == day, "demand"] = fit.predict(test_day[x])* 1.03
test = test.assign(id=test.id + "_" + np.where(test.d < FIRST + LENGTH, "validation", "evaluation"),
F="F" +(test.d - FIRST + 1 - LENGTH *(test.d >= FIRST + LENGTH)).astype("str"))
submission = test.pivot(index="id", columns="F", values="demand" ).reset_index() [cols].fillna(1)
 submission.to_csv(file, index=False, mode='w' if first else 'a', header=first)
 return True<save_to_csv>
DEBUG = False
SEED = 2021
INPUT_DIR = '.. /input/janestreet-save-as-feather/'
TRADING_THRESHOLD = 0.50
DATE_BEGIN = 86
CV_STRATEGY = 'PurgedGroupTimeSeriesSplit'
FOLDS = 5
THRESHOLD = 0.5
STAGE = 3
if STAGE == 2:
WEIGHT_DIR = '.. /input/janestreet-mlp-pretrain-stage1-weight-0/'
elif STAGE == 3:
WEIGHT_DIR = '.. /input/janestreet-mlp-train-stage2-weight-0/'
for i, dept in enumerate(np.unique(sales.dept_id)) :
train, valid, test, x = prep_data(sales, 1150, dept)
 fit = fit_model(train, valid, dept)
 pred_to_csv(fit, test, x, first=(i==0))<set_options>
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]
%matplotlib inline
%load_ext autoreload
%autoreload 2
!pip install wordcloud
!pip install transformers==3.0.2
!pip install simpletransformers
!pip install sklearn
!pip install nltk
!pip install unidecode
!pip install normalise
!pip install contractions
sys.path.insert(1, '/kaggle/input/pymodules4/')
 os.system('python3 -m spacy download en')
 train_df = pd.read_csv("/kaggle/input/data-baby2/train.csv")
 test_df = pd.read_csv("/kaggle/input/data-baby2/test.csv")
 msk = np.random.rand(len(train_df)) < 0.3
dev_df = train_df[msk]
train_df = train_df[~msk]
dev_df.reset_index(inplace=True)
 train_df.reset_index(inplace=True )<filter>
class GroupKFold(object):

 def __init__(self, n_splits=4, shuffle=True, random_state=42):
self.n_splits = n_splits
self.shuffle = shuffle
self.random_state = random_state
def get_n_splits(self, X=None, y=None, group=None):
return self.n_splits
def split(self, X, y, group):
kf = model_selection.KFold(n_splits=self.n_splits, shuffle=self.shuffle, random_state=self.random_state)
 unique_ids = X[group].unique()
for fold,(tr_group_idx, va_group_idx)in enumerate(kf.split(unique_ids)) :
tr_group, va_group = unique_ids[tr_group_idx], unique_ids[va_group_idx]
train_idx = np.where(X[group].isin(tr_group)) [0]
val_idx = np.where(X[group].isin(va_group)) [0]
yield train_idx, val_idx
class StratifiedGroupKFold(object):

 def __init__(self, n_splits=4, shuffle=True, random_state=42):
self.n_splits = n_splits
self.shuffle = shuffle
self.random_state = random_state
def get_n_splits(self, X=None, y=None, group=None):
return self.n_splits
def split(self, X, y, group):
labels_num = np.max(y)+ 1
y_counts_per_group = defaultdict(lambda: np.zeros(labels_num))
y_distr = Counter()
groups = X[group].values
for label, g in zip(y, groups):
y_counts_per_group[g][label] += 1
y_distr[label] += 1
y_counts_per_fold = defaultdict(lambda: np.zeros(labels_num))
groups_per_fold = defaultdict(set)
 def eval_y_counts_per_fold(y_counts, fold):
y_counts_per_fold[fold] += y_counts
std_per_label = []
for label in range(labels_num):
label_std = np.std([y_counts_per_fold[i][label] / y_distr[label] for i in range(self.n_splits)])
 std_per_label.append(label_std)
 y_counts_per_fold[fold] -= y_counts
return np.mean(std_per_label)
 groups_and_y_counts = list(y_counts_per_group.items())
 random.Random(self.random_state ).shuffle(groups_and_y_counts)
 for g, y_counts in sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])) :
best_fold = None
min_eval = None
for i in range(self.n_splits):
fold_eval = eval_y_counts_per_fold(y_counts, i)
 if min_eval is None or fold_eval < min_eval:
min_eval = fold_eval
best_fold = i
y_counts_per_fold[best_fold] += y_counts
groups_per_fold[best_fold].add(g)
 all_groups = set(groups)
 for i in range(self.n_splits):
train_groups = all_groups - groups_per_fold[i]
test_groups = groups_per_fold[i]
train_idx = [i for i, g in enumerate(groups)if g in train_groups]
test_idx = [i for i, g in enumerate(groups)if g in test_groups]
yield train_idx, test_idx
train_df[train_df["target"] == 0]["text"].values[1]<filter>
train = pd.read_feather('.. /input/janestreet-save-as-feather/train.feather' )
train_has_keyword_df = train_df[train_df.keyword.notnull() ]
train_has_location_df = train_df[train_df.location.notnull() ]
train_has_keyword_df.head()<string_transform>
original_size = train.shape[0]
train = train.query(f'date >= {DATE_BEGIN}')
 if STAGE == 1:
train = train.query('weight == 0' ).reset_index(drop=True)
 elif STAGE == 2:
train = train.query('weight > 0' ).reset_index(drop=True)
 elif STAGE == 3:
train = train.query('weight > 0' ).reset_index(drop=True)
 print('inference')
 print('Train size reduced from {:,} to {:,}.'.format(original_size, train.shape[0]))
text = " ".join(keyword for keyword in train_has_keyword_df.keyword)
 print("There are {} words in the combination of all keywords.".format(len(text)) )<string_transform>
train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
text = " ".join(keyword for keyword in train_has_keyword_df.keyword)
 renderWordcloud(text )<count_unique_values>
if DEBUG:
train = train.sample(10000, random_state=SEED)
 X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )
len(train_has_keyword_df.keyword.unique()), len(train_has_location_df.location.unique() )<groupby>
def seed_everything(seed : int)-> NoReturn :
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED )
group_keyword_sum_target = train_has_keyword_df.groupby("keyword" ).sum().sort_values("target")
 group_keyword_len = train_has_keyword_df.groupby("keyword" ).count()
group_keyword_sum_target<set_options>
nn_params = {
'hidden_unit': 256,
'num_layers': 3,
'gaussnoise': 0.01,
'dropout': 0.2,
'learning_rate': 1e-03,
'label_smoothing': 1e-02,
'batch_size': 1024,
'epochs': 196
}
def create_model(nn_params, input_dim, output_dim):
inputs = tf.keras.layers.Input(input_dim)
 x = tf.keras.layers.BatchNormalization()(inputs)
 for i in range(nn_params['num_layers']):
x = tf.keras.layers.Dense(nn_params['hidden_unit'] //(2 ** i), activation='relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.GaussianNoise(nn_params['gaussnoise'] )(x)
 x = tf.keras.layers.Dropout(nn_params['dropout'] )(x)
 x = tf.keras.layers.Dense(output_dim, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=inputs, outputs=x)
 opt = tf.keras.optimizers.Adam(learning_rate=nn_params['learning_rate'])
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=nn_params['label_smoothing'])
 model.compile(optimizer=opt,
loss=loss,
metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model
model = create_model(nn_params, 130, 5)
 model.summary()
pd.set_option('display.max_rows', None)
 group_keyword_sum_target<groupby>
%%time
if STAGE == 3:
models = []
for fold in tqdm(range(FOLDS)) :
tf.keras.backend.clear_session()
model = create_model(nn_params, X.shape[-1], y.shape[-1])
 model.load_weights(WEIGHT_DIR + f'model_{SEED}_{fold}.hdf5')
 models.append(model)
 else:
if CV_STRATEGY == 'PurgedGroupTimeSeriesSplit':
gkf = PurgedGroupTimeSeriesSplit(n_splits=FOLDS, group_gap=20)
 splits = list(gkf.split(y, groups=train['date'].values))
elif CV_STRATEGY == "GroupKFold":
cv = GroupKFold(n_splits=FOLDS, shuffle=True, random_state=SEED)
 splits = cv.split(train, train['resp'].values.astype(int), 'date')
 elif CV_STRATEGY == "StratifiedGroupKFold":
cv = StratifiedGroupKFold(n_splits=FOLDS, shuffle=True, random_state=SEED)
 splits = cv.split(train, train['resp'].values.astype(int), 'date')
 models = []
for fold,(train_indices, test_indices)in tqdm(enumerate(splits)) :
X_train, X_test = X[train_indices], X[test_indices]
y_train, y_test = y[train_indices], y[test_indices]
tf.keras.backend.clear_session()
model = create_model(nn_params, X.shape[-1], y.shape[-1])
 if STAGE == 2:
model.load_weights(WEIGHT_DIR + f'model_{SEED}_{fold}.hdf5')
 er = tf.keras.callbacks.EarlyStopping(patience=8, restore_best_weights=True, monitor='val_loss')
 ReduceLR = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=8, verbose=1, mode='min')
 model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=f'./model_{SEED}_{fold}.hdf5',
save_weights_only=True, verbose=0, monitor='val_loss', save_best_only=True)
 nn_callbacks = [er, ReduceLR, model_checkpoint_callback]
model.fit(X_train, y_train, validation_data=(X_test,y_test),
epochs=nn_params['epochs'], batch_size=nn_params['batch_size'], callbacks=nn_callbacks)
 models.append(model )
<sort_values><EOS> 
if STAGE == 3:
f = np.median
models = models[-3:]
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= THRESHOLD, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df)
 else:
print('Training done!' )