Kaggle competition: Jane Street Market Prediction
submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission_BERT.csv', index=False )<import_modules>
def save_pickle(dict_param,nom_fich):
with open(nom_fich,"wb")as f :
pickle.dump(dict_param,f )
import numpy as np
import pandas as pd
from sklearn import ensemble, neighbors, linear_model, metrics, preprocessing
from datetime import datetime
import glob, re<load_from_csv>
path = "/kaggle/input/jane-street-market-prediction/"
train = pd.read_csv(path + "train.csv")
 features = pd.read_csv(path + "features.csv" )
data = {
'ar': pd.read_csv('.. /input/air_reserve.csv'),
'as': pd.read_csv('.. /input/air_store_info.csv'),
'tra': pd.read_csv('.. /input/air_visit_data.csv'),
'hol': pd.read_csv('.. /input/date_info.csv' ).rename(columns={'calendar_date': 'visit_date'}),
'hr': pd.read_csv('.. /input/hpg_reserve.csv'),
'hs': pd.read_csv('.. /input/hpg_store_info.csv'),
'tes': pd.read_csv('.. /input/sample_submission.csv'),
'id': pd.read_csv('.. /input/store_id_relation.csv')
 }<merge>
missing_table = pd.DataFrame({c:(train[c].isna().sum() /len(train)) *100 for c in train.columns},index=["% missing values"])
 missing_table
data['hr'] = data['hr'].merge(data['id'], how='inner', on=['hpg_store_id'])
 data['hr'].sample(5 )<data_type_conversions>
def detect_correlated_features(df,threshold=0.5):

 correlated= defaultdict(list)
 for col in df.columns:
dex = list(df.columns ).index(col)
 for ind in df.index[dex+1:] :
if df.loc[col,ind] > threshold:
correlated[col].append(ind)
 return correlated
for df in ['ar', 'hr']:
data[df]['visit_datetime'] = pd.to_datetime(data[df]['visit_datetime'])
 data[df]['visit_datetime'] = data[df]['visit_datetime'].dt.date
data[df]['reserve_datetime'] = pd.to_datetime(data[df]['reserve_datetime'])
 data[df]['reserve_datetime'] = data[df]['reserve_datetime'].dt.date
data[df]['reserve_datetime_diff'] = data[df].apply(
 lambda r:(r['visit_datetime'] - r['reserve_datetime'] ).days, axis=1)
 data[df] = data[df].groupby(['air_store_id', 'visit_datetime'], as_index=False)[['reserve_datetime_diff',
'reserve_visitors']].sum().rename(columns={'visit_datetime':'visit_date'})
 data['hr'].sample(5 )<categorify>
correlated_features = detect_correlated_features(correlation_table,threshold=0.9 )
lbl = preprocessing.LabelEncoder()
data['as']['air_genre_name'] = lbl.fit_transform(data['as']['air_genre_name'])
 data['as']['air_area_name'] = lbl.fit_transform(data['as']['air_area_name'])
 data['as'].sample(3 )<feature_engineering>
features_to_drop = [f for f in ay_features if f not in ax_features]
data['tra']['visit_date'] = pd.to_datetime(data['tra']['visit_date'])
 data['tra']['day_of_week'] = data['tra']['visit_date'].dt.dayofweek
data['tra']['month'] = data['tra']['visit_date'].dt.month
data['tra']['year'] = data['tra']['visit_date'].dt.year
data['tra']['visit_date'] = data['tra']['visit_date'].dt.date 
<data_type_conversions>
train_df = train[[ f for f in list(train.columns)if(( f not in features_to_drop)or(f =="resp")) ]]
data['hol'] = data['hol'].drop(columns=['day_of_week'])
 data['hol']['visit_date'] = pd.to_datetime(data['hol']['visit_date'])
 data['hol']['visit_date'] = data['hol']['visit_date'].dt.date
data['hol'].sample(4 )<feature_engineering>
label_correlation = pd.DataFrame({c:train_df["resp"].corr(train_df[c])for\
c in train_df.columns if c!="resp" and c!="feature_0"},index=["action"])
data['tes']['visit_date'] = data['tes']['id'].map(lambda x: str(x ).split('_')[2])
 data['tes']['air_store_id'] = data['tes']['id'].map(lambda x: '_'.join(x.split('_')[:2]))
data['tes']['visit_date'] = pd.to_datetime(data['tes']['visit_date'])
 data['tes']['day_of_week'] = data['tes']['visit_date'].dt.dayofweek
data['tes']['month'] = data['tes']['visit_date'].dt.month
data['tes']['year'] = data['tes']['visit_date'].dt.year
data['tes']['visit_date'] = data['tes']['visit_date'].dt.date
data['tes'].sample(4 )<merge>
train_df.loc[:,"target"] = list(((train_df["resp"] > 0)&(train_df["resp_1"] >0)&(train_df["resp_2"]>0)\
&(train_df["resp_3"]>0)&(train_df["resp_4"]>0)).astype("int"))
train_df.loc[:,"vl"] = list(train_df["target"].values)
 pvt_table=train_df[["feature_0","target"]].pivot_table(index=["feature_0"],columns=["target"],aggfunc=len)
 tx= train_df["feature_0"].value_counts()
ty = train_df["target"].value_counts()
tx = pd.DataFrame(tx)
 ty = pd.DataFrame(ty)
 n = len(train_df)
 tx.columns = ["values"]
ty.columns = ["values"]
cnt = tx.dot(ty.T)/n
ind = cnt.index
pvt_table = pvt_table.loc[ind,:]
mesure =(cnt - pvt_table)**2/cnt
xin = mesure.sum().sum()
del(train_df["target"])
 del(train_df["vl"])
 fig = plt.figure(figsize=(12,8))
sns.heatmap(mesure,annot=True,linewidths=0.5,cmap="BrBG",vmin=0,vmax=1)
 plt.title("Correlation table between feature_0 and the target value ",size=15,color="red")
 print("The total correlation between feature_0 and the target equal to {}".format(xin))
train = pd.merge(data['tra'], data['hol'], how='left', on=['visit_date'])
 train = pd.merge(data['tra'], stores, how='left', on=['air_store_id', 'day_of_week'] )<merge>
missing_b_imputation = pd.DataFrame({c:(train_df[c].isna().sum())for c in train_df.columns},index=["% missing values"])
 print("The number of missing values in the train_df dataframe before imputation processing :{}".format(\
missing_b_imputation.sum().sum()))
missing_b_imputation
test = pd.merge(data['tes'], data['hol'], how='left', on=['visit_date'])
 test = pd.merge(data['tes'], stores, how='left', on=['air_store_id', 'day_of_week'])
 test.sample(3 )<merge>
features_with_missing_values = []
for f in list(train_df.columns):
if missing_table.loc["% missing values",f] > 0 :
features_with_missing_values.append(f )
data['ar'] = data['ar'].merge(data['hr'], how='left', on=['air_store_id', 'visit_date', 'reserve_datetime_diff', 'reserve_visitors'])
 data['ar'].sample(4 )<merge>
for f in features_with_missing_values :
model = LinearRegression()
if len(correlated_features[f])> 0 :
correlated = correlated_features[f][0]
if correlated in train.columns :
model.fit(train.loc[(train[correlated].notna())&(train[f].notna()),correlated].values.reshape(-1,1),\
train.loc[(train[correlated].notna())&(train[f].notna()),f])
 values_to_impute = train_df.loc[(train[f].isna())&(train[correlated].notna()),f]
imputer = train.loc[(train[f].isna())&(train[correlated].notna()),correlated].values
if(len(values_to_impute)> 0)&(len(imputer)> 0):
train_df.loc[(train[f].isna())&(train[correlated].notna()),f] = model.predict(train.loc[(train[f].isna())&(train[correlated].notna()),correlated].values.\
reshape(-1,1))
train = train.merge(data['ar'], how='left', on=['air_store_id', 'visit_date'])
 test = test.merge(data['ar'], how='left', on=['air_store_id', 'visit_date'] )<categorify>
missing_a_imputation = pd.DataFrame({c:(train_df[c].isna().sum())for c in train_df.columns},index=["Number missing values"])
 print("The number of missing values in the train_df dataframe after imputation processing :{}".format(\
missing_a_imputation.sum().sum()))
missing_a_imputation
col = [c for c in train if c not in ['id', 'air_store_id', 'visit_date', 'visitors']]
train = train.fillna(-1)
 test = test.fillna(-1)
 train.sample(4 )<compute_test_metric>
for f in features_with_missing_values:
train_df.fillna(train_df[f].mean() ,inplace=True )
def RMSLE(y, pred):
return metrics.mean_squared_error(y, pred)**0.5<train_model>
train_df["action"] =(( train_df["resp"] > 0)&(train_df["resp_1"] >0)&(train_df["resp_2"]>0)\
&(train_df["resp_3"]>0)&(train_df["resp_4"]>0)).astype("int")
etc = ensemble.ExtraTreesRegressor(n_estimators=225, max_depth=5, n_jobs=-1, random_state=3)
 knn = neighbors.KNeighborsRegressor(n_jobs=-1, n_neighbors=4)
 etc.fit(train[col], np.log1p(train['visitors'].values))
knn.fit(train[col], np.log1p(train['visitors'].values))<predict_on_test>
features = pd.read_csv(path + "features.csv" )
test['visitors'] =(etc.predict(test[col])/ 2)+(knn.predict(test[col])/ 2)
 test['visitors'] = np.expm1(test['visitors'] ).clip(lower=0.)
sub1 = test[['id', 'visitors']].copy()<load_from_csv>
features.set_index('feature',inplace=True)
 features = features.astype("int8" )
dfs = {re.search('/([^/\.]*)\.csv', fn ).group(1):
pd.read_csv(fn)for fn in glob.glob('.. /input/*.csv')}
for k, v in dfs.items() : locals() [k] = v<feature_engineering>
t_sne = tsne(n_components=2,random_state=42 ).fit_transform(features.values )
date_info['weight'] =(( date_info.index + 1)/ len(date_info)) ** 5<merge>
clusters_number = [4,8,12,16,20,24,28]
silhouette_performances = {}
for cl_n in clusters_number :
kmeans = KMeans(n_clusters=cl_n)
 kmeans.fit(features.values)
 sc=silhouette_score(features.values,kmeans.labels_)
 silhouette_performances[sc] = cl_n
visit_data = air_visit_data.merge(date_info, left_on = 'visit_date', right_on = 'calendar_date', how = 'left')
 visit_data.drop('calendar_date', axis=1, inplace = True)
 visit_data['visitors'] = visit_data.visitors.map(pd.np.log1p)
 visit_data.head()<groupby>
best_model = KMeans(n_clusters=20)
 best_model.fit(features.values )
wmean = lambda x:(( x.weight * x.visitors ).sum() / x.weight.sum())
 visitors = visit_data.groupby(['air_store_id', 'day_of_week', 'holiday_flg'] ).apply(wmean ).rename('visitors' ).reset_index()
visitors.head()<merge>
train_df = train_df.query('date > 85' ).reset_index(drop = True )
sample_submission['air_store_id'] = sample_submission.id.map(lambda x: '_'.join(x.split('_')[:-1]))
sample_submission['calendar_date'] = sample_submission.id.map(lambda x: x.split('_')[2])
 sample_submission.drop('visitors', axis=1, inplace=True)
 sample_submission = sample_submission.merge(date_info, on='calendar_date', how='left')
 sample_submission = sample_submission.merge(visitors, on=[
'air_store_id', 'day_of_week', 'holiday_flg'], how='left' )<count_missing_values>
features = list(train_df.columns)
 features.remove("weight")
 features.remove("resp_1")
 features.remove("resp_2")
 features.remove("resp_3")
 features.remove("resp_4")
 features.remove("resp")
 features.remove("action")
 features.remove("ts_id")
 features.remove("date")
 features.remove("feature_0")
sample_submission.visitors.isnull().sum()<merge>
save_pickle(features,"features_names" )
missings = sample_submission.visitors.isnull()
sample_submission.loc[missings, 'visitors'] = sample_submission[missings].merge(
 visitors[visitors.holiday_flg == 0], on=('air_store_id', 'day_of_week'),
how='left')['visitors_y'].values<count_missing_values>
abundant_class = train_df.loc[(train_df["weight"] > 0)&(train_df["action"]==0),:]
rare_class = train_df.loc[(train_df["weight"] > 0)&(train_df["action"]==1),:]
sample_submission.isnull().sum()<merge>
imputer = np.mean(train_df[features].values,axis=0)
 save_pickle(imputer,"features_imputation" )
missings = sample_submission.visitors.isnull()
sample_submission.loc[missings, 'visitors'] = sample_submission[missings].merge(
 visitors[['air_store_id', 'visitors']].groupby('air_store_id' ).mean().reset_index() ,
on='air_store_id', how='left')['visitors_y'].values<count_missing_values>
l = len(rare_class)
 df1 = abundant_class.iloc[:l,:].append(rare_class)
 df2 = abundant_class.iloc[l:2*l,:].append(rare_class)
 df3 = abundant_class.iloc[2*l:,:].append(rare_class)
 df1 = df1.sample(frac=1)
 df2 = df2.sample(frac=1)
 df3 = df3.sample(frac=1 )
sample_submission.isnull().sum()<merge>
del(train_df)
 del(train )
sample_submission['visitors'] = sample_submission.visitors.map(pd.np.expm1)
 sub2 = sample_submission[['id', 'visitors']].copy()
sub_merge = pd.merge(sub1, sub2, on='id', how='inner' )<save_to_csv>
training= [df1[features],df2[features],df3[features]]
targets = [df1["action"],df2["action"],df3["action"]]
sub_merge['visitors'] =(sub_merge['visitors_x'] * sub_merge['visitors_y'])**(1/2)
 sub_merge[['id', 'visitors']].to_csv('sub_geo_mean.csv', index = False)
 sub_merge[['id', 'visitors']].head()<save_to_csv>
datas = []
for i in range(3):
xtr,xval,ytr,yval = train_test_split(training[i].values,targets[i].values,test_size=0.1,\
stratify=targets[i].values)
 datas.append(((xtr,ytr),(xval,yval)) )
sub_merge['visitors'] = 2/(1/sub_merge['visitors_x'] + 1/sub_merge['visitors_y'])
 sub_merge[['id', 'visitors']].to_csv('sub_hrm_mean.csv', index = False)
 sub_merge[['id', 'visitors']].head()<set_options>
del(df1)
 del(df2)
 del(df3)
 del(abundant_class)
 del(rare_class)
 del(features_with_missing_values)
 del(correlation_table)
 del(missing_table )
%reload_ext autoreload
%autoreload 2
%matplotlib inline
package_dir = '.. /input/efficientnet/efficientnet-pytorch/EfficientNet-PyTorch/'
sys.path.insert(0, package_dir)
 start = time.time()<feature_engineering>
gc.collect()
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 999
seed_everything(SEED )<load_from_csv>
params={"num_leaves":300,
"max_bin":450,
"feature_fraction":0.52,
"bagging_fraction":0.52,
"objective":"binary",
"learning_rate":0.05,
"boosting_type":"gbdt",
"metric":"auc"
}
models = []
for i in range(3):
xtr = datas[i][0][0]
ytr = datas[i][0][1]
xval = datas[i][1][0]
yval = datas[i][1][1]
d_train = lgbm.Dataset(xtr,label=ytr)
 d_eval = lgbm.Dataset(xval,label=yval,reference=d_train)
 clf = lgbm.train(params,d_train,valid_sets=[d_train,d_eval],num_boost_round=1500,\
early_stopping_rounds=50,verbose_eval=50)
 clf.save_model("weights_{}".format(i))
models.append(clf )
deployment_dir = ".. /input/efficientnetstacking"
def qk(y_pred, y):
k = torch.tensor(cohen_kappa_score(torch.round(y_pred), y, weights='quadratic'), device='cuda:0')
 k[k != k] = 0
k[torch.isinf(k)] = 0
return k
test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv' )<load_pretrained>
env = janestreet.make_env()
inf1_t1 = time.time()
b3_models = ["efficientnet-b3_0901_16-45-51_stage2_f1", "efficientnet-b3_0901_16-45-51_stage2_f2",
"efficientnet-b3_0901_16-45-51_stage2_f3", "efficientnet-b3_0901_16-45-51_stage2_f4",
"efficientnet-b3_0901_16-45-51_stage2_f5"]
b3_test_logits_list = []
for m in b3_models:
learn = load_learner(deployment_dir, "{}.pkl".format(m))
learn.data.add_test(ImageList.from_df(test_df,
'.. /input',
folder='test_images_ben_preprocessing_sigmaX10',
suffix='.png'))
logits,_ = learn.get_preds(DatasetType.Test)
 logits = logits.numpy()
b3_test_logits_list.append(logits)
 print(logits.shape)
 b4_models = ["efficientnet-b4_0820_01-09-57_stage2_f1", "efficientnet-b4_0820_01-09-57_stage2_f2",
"efficientnet-b4_0820_01-09-57_stage2_f3", "efficientnet-b4_0820_01-09-57_stage2_f4",
"efficientnet-b4_0821_00-02-25_stage2_f5"]
b4_test_logits_list = []
for m in b4_models:
learn = load_learner(deployment_dir, "{}.pkl".format(m))
learn.data.add_test(ImageList.from_df(test_df,
'.. /input',
folder='test_images_ben_preprocessing_sigmaX10',
suffix='.png'))
logits,_ = learn.get_preds(DatasetType.Test)
 logits = logits.numpy()
b4_test_logits_list.append(logits)
 print(logits.shape)
 b5_models = ["efficientnet-b5_0820_01-32-30_stage2_f1", "efficientnet-b5_0903_01-03-41_stage2_f2",
"efficientnet-b5_0820_22-13-07_stage2_f3", "efficientnet-b5_0821_01-30-37_stage2_f4",
"efficientnet-b5_0821_00-26-51_stage2_f5"]
b5_test_logits_list = []
for m in b5_models:
learn = load_learner(deployment_dir, "{}.pkl".format(m))
learn.data.add_test(ImageList.from_df(test_df,
'.. /input',
folder='test_images_ben_preprocessing_sigmaX10',
suffix='.png'))
logits,_ = learn.get_preds(DatasetType.Test)
 logits = logits.numpy()
b5_test_logits_list.append(logits)
 print(logits.shape)
 with open(os.path.join(deployment_dir, "xgboost-0903_05-26-03.pkl"), "rb")as f:
xlf = pk.load(f)
 with open(os.path.join(deployment_dir, "svr-0903_05-26-03.pkl"), "rb")as f:
svr = pk.load(f)
 with open(os.path.join(deployment_dir, "cb-0906_06-09-42.pkl"), "rb")as f:
cb = pk.load(f)
 results_xlf = []
results_svr = []
results_cb = []
for b3, b4, b5 in zip(b3_test_logits_list, b4_test_logits_list, b5_test_logits_list):
X_test = np.concatenate([b3, b4, b5], axis=1)
 res_xlf = xlf.predict(X_test)
 results_xlf.append(res_xlf)
 res_svr = svr.predict(X_test)
 results_svr.append(res_svr)
 res_cb = cb.predict(X_test)
 results_cb.append(res_cb)
 avg_res_xlf = np.average(results_xlf, axis=0)
 avg_res_svr = np.average(results_svr, axis=0)
 avg_res_cb = np.average(results_cb, axis=0)
 y_pred_xlf = np.round(avg_res_xlf ).astype(int)
 y_pred_svr = np.round(avg_res_svr ).astype(int)
 y_pred_cb = np.round(avg_res_cb ).astype(int)
 test_df.diagnosis = y_pred_xlf
test_df.hist()
plt.show()
test_df.diagnosis = y_pred_svr
test_df.hist()
plt.show()
test_df.diagnosis = y_pred_cb
test_df.hist()
plt.show()
inf1_t2 = time.time()
inf1_dur = inf1_t2 - inf1_t1
print('Inference duration ',inf1_dur, inf1_dur/3600 )<compute_test_metric>
th = 0.5000
<compute_test_metric><EOS> 
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df["weight"].item() > 0 :
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt.sum()):
x_tt = np.nan_to_num(x_tt)+ np.isnan(x_tt)* imputer
pred = np.mean(np.stack([model.predict(x_tt)for model in models]),axis=0 ).T
pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else :
pred_df.action = 0
env.predict(pred_df )