Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<create_dataframe>
!pip --quiet install.. /input/treelite/treelite-0.93-py3-none-manylinux2010_x86_64.whl
missings_count = {col:comp_pop_sel[col].isnull().sum() for col in comp_pop_sel.columns}
missings = pd.DataFrame.from_dict(missings_count, orient='index')
 print(missings.nlargest(30, 0))<data_type_conversions>
!pip --quiet install.. /input/treelite/treelite_runtime-0.93-py3-none-manylinux2010_x86_64.whl
comp_pop_sel['Population'] = comp_pop_sel['Population'].fillna(0 )<filter>
sns.set_context("talk")
 style.use('fivethirtyeight')
 pd.options.display.max_columns = None
warnings.filterwarnings('ignore' )
comp_df_Italy = comp_pop_sel[comp_pop_sel['Country/Region']=='Italy']
comp_df_Italy = comp_df_Italy.reset_index(drop = True)
 comp_df_Italy.tail(5 )<categorify>
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]
comp_df_Italy['Day_num'] = preprocessing.LabelEncoder().fit_transform(comp_df_Italy.Date)
 comp_df_Italy.tail(3 )<feature_engineering>
SEED = 42
START_DATE = 85
INPUT_DIR = '.. /input/janestreet-save-as-feather/'
TRADING_THRESHOLD = 0.502
comp_df_Italy['R'] = comp_df_Italy['Deaths']+comp_df_Italy['Recovered']
comp_df_Italy['I'] = comp_df_Italy['Confirmed']- comp_df_Italy['R']
comp_df_Italy['S'] = comp_df_Italy['Population']-comp_df_Italy['I']-comp_df_Italy['R']
comp_df_Italy.tail(3 )<categorify>
%%time
def load_data(input_dir=INPUT_DIR):
train = pd.read_feather(pathlib.Path(input_dir + 'train.feather'))
return train
train = load_data(INPUT_DIR )
comp_df_Italy_S = comp_df_Italy['S']
comp_df_Italy_I = comp_df_Italy['I']
comp_df_Italy_R = comp_df_Italy['R']
comp_df_Italy_s = np.array(comp_df_Italy_S, dtype=float)
 comp_df_Italy_i = np.array(comp_df_Italy_I, dtype=float)
 comp_df_Italy_r = np.array(comp_df_Italy_R, dtype=float)
 display(comp_df_Italy_s[9],comp_df_Italy_i[9], comp_df_Italy_r[9] )<load_from_csv>
train = train.query(f'date > {START_DATE}')
 train.fillna(train.mean() ,inplace=True)
 train = train[train['weight'] != 0]
df_train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 df_subm = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )<create_dataframe>
features = train.columns[train.columns.str.startswith('feature')].values.tolist()
print('{} features used'.format(len(features)) )
df_train_copy = df_train.copy()
df_test_copy = df_test.copy()<data_type_conversions>
train['action'] =(train['resp'] > 0 ).astype('int')
 f_mean = np.mean(train[features[1:]].values,axis=0 )
df_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province_State':'State'}, inplace=True)
 df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_test.rename(columns={'Province_State':'State'}, inplace=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True )<prepare_x_and_y>
params = {'n_estimators': 473, 'max_depth': 7, 'min_child_weight': 6,
'learning_rate': 0.015944928866056352, 'subsample': 0.608128483148888,
'gamma': 0, 'colsample_bytree': 0.643875232059528,'objective':'binary:logistic',
'eval_metric': 'auc','tree_method': 'gpu_hist', 'random_state': 42,}
params_1 = {'n_estimators': 494, 'max_depth': 8, 'min_child_weight': 6, 'learning_rate': 0.009624384025871735,
'subsample': 0.8328412036014541, 'gamma': 0, 'colsample_bytree': 0.715303237773365,
'objective':'binary:logistic', 'eval_metric': 'auc','tree_method': 'gpu_hist', 'random_state': 42,}
y1_df_Train = df_train.iloc[:, -2]
y2_df_Train = df_train.iloc[:, -1]
NOVAL = "NOVAL"
def handlenullstate(State, Country):
if State == NOVAL: return Country
return State<data_type_conversions>
training = True
if training:
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
groups = train['date'].values
models = []
scores = []
cv = PurgedGroupTimeSeriesSplit(
 n_splits=4,
group_gap=20, 
)
 for t in tqdm(range(y.shape[1])) :
yy = y[:,t]
for i,(train_index, valid_index)in enumerate(cv.split(
 X,
yy,
groups=groups)) :
print(f'Target {t} Fold {i} started at {time.ctime() }')
 X_train, X_valid = X[train_index], X[valid_index]
y_train, y_valid = yy[train_index], yy[valid_index]
model = xgb.XGBClassifier(**params_1, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_valid, y_valid)], eval_metric='auc',
verbose=100, callbacks = [xgb.callback.EarlyStopping(rounds=300,save_best=True)])
 pred = model.predict(X_valid)
 score = roc_auc_score(y_valid,pred)
 model.save_model(f'my_model_{t}_{i}.model')
 models.append(model)
 scores.append(score)
 del score, model
print(scores)
 del X_train, X_valid, y_train, y_valid
rubbish = gc.collect()
df_train['State'].fillna(NOVAL, inplace=True)
 df_train['State'] = df_train.loc[:, ['State', 'Country']].apply(lambda x : handlenullstate(x['State'], x['Country']), axis=1)
 df_train.loc[:, 'Date'] = df_train.Date.dt.strftime("%m%d")
 df_train["Date"] = df_train["Date"].astype(int)
 df_test['State'].fillna(NOVAL, inplace=True)
 df_test['State'] = df_test.loc[:, ['State', 'Country']].apply(lambda x : handlenullstate(x['State'], x['Country']), axis=1)
 df_test.loc[:, 'Date'] = df_test.Date.dt.strftime("%m%d")
 df_test["Date"] = df_test["Date"].astype(int)
 df_test.head()<define_variables>
if training:
model_0 = treelite.Model.load('my_model_0_3.model', model_format='xgboost')
 model_1 = treelite.Model.load('my_model_1_3.model', model_format='xgboost')
 model_2 = treelite.Model.load('my_model_2_3.model', model_format='xgboost')
 model_3 = treelite.Model.load('my_model_3_3.model', model_format='xgboost')
 model_4 = treelite.Model.load('my_model_4_3.model', model_format='xgboost' )
covid_pop_taken = covid_pop[['Province.State','Country.Region','Population']]<rename_columns>
if training:
m = [model_0,model_1,model_2,model_3,model_4]
for j,i in enumerate(m):
toolchain = 'gcc'
i.export_lib(toolchain=toolchain, libpath=f'./mymodel_{j}.so',
params={'parallel_comp': 32}, verbose=True )
covid_pop_taken.rename(columns={'Province.State':'State'}, inplace=True)
 covid_pop_taken.rename(columns={'Country.Region':'Country'}, inplace=True )<feature_engineering>
if training:
predictor_0 = treelite_runtime.Predictor(f'./mymodel_{0}.so', verbose=True)
 predictor_1 = treelite_runtime.Predictor(f'./mymodel_{1}.so', verbose=True)
 predictor_2 = treelite_runtime.Predictor(f'./mymodel_{2}.so', verbose=True)
 predictor_3 = treelite_runtime.Predictor(f'./mymodel_{3}.so', verbose=True)
 predictor_4 = treelite_runtime.Predictor(f'./mymodel_{4}.so', verbose=True )
covid_pop_taken['State'].fillna("None", inplace=True)
 NOVAL = 'None'
covid_pop_taken['State'] = covid_pop_taken.loc[:, ['State', 'Country']].apply(lambda x : handlenullstate(x['State'], x['Country']), axis=1)
 covid_pop_taken.head(3 )<prepare_output>
env = janestreet.make_env()
iter_test = env.iter_test()
<merge><EOS> 
f = np.median
index_features = [n for n in range(1,(len(features)+ 1)) ]
for(test_df, pred_df)in tqdm(iter_test):
if test_df['weight'].item() > 0:
x_tt = test_df.values[0][index_features].reshape(1,-1)
 if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
batch = treelite_runtime.Batch.from_npy2d(x_tt)
 pred_0 = predictor_0.predict(batch)
 pred_1 = predictor_1.predict(batch)
 pred_2 = predictor_2.predict(batch)
 pred_3 = predictor_3.predict(batch)
 pred_4 = predictor_4.predict(batch)
 pred = np.stack([pred_0,pred_1,pred_2,pred_3,pred_4],axis=0 ).T
pred = f(pred)
 pred_df.action = int(pred >= TRADING_THRESHOLD)
 else:
pred_df['action'].values[0] = 0
env.predict(pred_df )