Kaggle competition: Penyisihan Datavidia 2019
<SOS> metric: meanfscore Kaggle data source: penyisihan-datavidia-2019<feature_engineering>
def get_metrics(x, y, e):
z = e.predict(x)
 print(classification_report(z, y, digits=4))
all_data['_healthItems'] = all_data['heals'] + all_data['boosts']
all_data['_headshotKillRate'] = all_data['headshotKills'] / all_data['kills']
all_data['_killPlaceOverMaxPlace'] = all_data['killPlace'] / all_data['maxPlace']
all_data['_killsOverWalkDistance'] = all_data['kills'] / all_data['walkDistance']
fillInf(all_data, 0 )<drop_column>
warnings.filterwarnings('ignore' )
all_data.drop(['boosts','heals','killStreaks','DBNOs'], axis=1, inplace=True)
 all_data.drop(['headshotKills','roadKills','vehicleDestroys'], axis=1, inplace=True)
 all_data.drop(['rideDistance','swimDistance','matchDuration'], axis=1, inplace=True)
 all_data.drop(['rankPoints','killPoints','winPoints'], axis=1, inplace=True )<groupby>
flight = pd.read_csv(".. /input/datavidia2019/flight.csv")
 test = pd.read_csv(".. /input/datavidia2019/test.csv")
 hotel = pd.read_csv(".. /input/datavidia2019/hotel.csv" )
match = all_data.groupby(['matchId'])
 group = all_data.groupby(['matchId','groupId','matchType'])
 agg_col = list(all_data.columns)
 exclude_agg_col = ['Id','matchId','groupId','matchType','maxPlace','numGroups','winPlacePerc']
for c in exclude_agg_col:
agg_col.remove(c)
 print(agg_col)
 sum_col = ['kills','killPlace','damageDealt','walkDistance','_healthItems']<merge>
def preprocess_city_trx(df):
df["v_city_proc"] = df.visited_city.map(lambda x: ast.literal_eval(x.strip("''")))
 df["log_trx_proc"] = df.log_transaction.map(lambda x: ast.literal_eval(x.strip("''")))
 return df
preprocess_city_trx(test)
 preprocess_city_trx(flight ).head()
minKills = all_data.sort_values(['matchId','groupId','kills','killPlace'] ).groupby(
 ['matchId','groupId','kills'] ).first().reset_index().copy()
for n in np.arange(4):
c = 'kills_' + str(n)+ '_Place'
nKills =(minKills['kills'] == n)
 minKills.loc[nKills, c] = minKills[nKills].groupby(['matchId'])['killPlace'].rank().values
match_data = pd.merge(match_data, minKills[nKills][['matchId','groupId',c]], how='left')
 match_data = reduce_mem_usage(match_data)
 del minKills, nKills
print(match_data.shape )<merge>
missing_data = pd.DataFrame({'total_missing': flight.isnull().sum() , 'perc_missing':(flight.isnull().sum() /len(flight.index)) *100})
 missing_data
all_data = pd.merge(all_data, match_data)
 del match_data
gc.collect()
all_data['enemy.players'] = all_data['m.players'] - all_data['players']
for c in sum_col:
all_data['p.max_msum.' + c] = all_data['max.' + c] / all_data['m.sum.' + c]
all_data['p.max_mmax.' + c] = all_data['max.' + c] / all_data['m.max.' + c]
all_data.drop(['m.sum.' + c, 'm.max.' + c], axis=1, inplace=True)
 fillInf(all_data, 0)
 print(all_data.shape )<groupby>
missing_data2 = pd.DataFrame({'total_missing': test.isnull().sum() , 'perc_missing':(test.isnull().sum() /len(test.index)) *100})
 missing_data2
match = all_data.groupby('matchId')
 matchRank = match[numcols].rank(pct=True ).rename(columns=lambda s: 'rank.' + s)
 all_data = reduce_mem_usage(pd.concat([all_data, matchRank], axis=1))
rank_col = matchRank.columns
del matchRank
gc.collect()
match = all_data.groupby('matchId')
 matchRank = match[rank_col].max().rename(columns=lambda s: 'max.' + s ).reset_index()
all_data = pd.merge(all_data, matchRank)
 for c in numcols:
all_data['rank.' + c] = all_data['rank.' + c] / all_data['max.rank.' + c]
all_data.drop(['max.rank.' + c], axis=1, inplace=True)
 del matchRank
gc.collect()
print(all_data.shape )<merge>
def get_is_cross_sell(df):
df["is_cross_sell"] = df.hotel_id.map(lambda x: 0 if x == "None" else 1)
 return df
get_is_cross_sell(flight ).is_cross_sell.value_counts()
killMinorRank = all_data[['matchId','min.kills','max.killPlace']].copy()
group = killMinorRank.groupby(['matchId','min.kills'])
 killMinorRank['rank.minor.maxKillPlace'] = group.rank(pct=True ).values
all_data = pd.merge(all_data, killMinorRank)
 killMinorRank = all_data[['matchId','max.kills','min.killPlace']].copy()
group = killMinorRank.groupby(['matchId','max.kills'])
 killMinorRank['rank.minor.minKillPlace'] = group.rank(pct=True ).values
all_data = pd.merge(all_data, killMinorRank)
 del killMinorRank
gc.collect()<count_unique_values>
flight["route"].value_counts()
constant_column = [col for col in all_data.columns if all_data[col].nunique() == 1]
print('drop columns:', constant_column)
 all_data.drop(constant_column, axis=1, inplace=True )<categorify>
flight['order_id'].nunique() , flight.shape[0]
all_data['matchType'] = all_data['matchType'].apply(mapper)
 all_data = pd.concat([all_data, pd.get_dummies(all_data['matchType'])], axis=1)
 all_data.drop(['matchType'], axis=1, inplace=True)
 all_data['matchId'] = all_data['matchId'].apply(lambda x: int(x,16))
all_data['groupId'] = all_data['groupId'].apply(lambda x: int(x,16))<sort_values>
flight.visited_city.value_counts()
null_cnt = all_data.isnull().sum().sort_values()
print(null_cnt[null_cnt > 0] )<feature_engineering>
y = flight.copy()
y["n_trx"] = y.log_trx_proc.map(len)
 y[y.n_trx == 2][y.trip == "trip"].visited_city.value_counts()
cols = [col for col in all_data.columns if col not in ['Id','matchId','groupId']]
for i, t in all_data.loc[:, cols].dtypes.iteritems() :
if t == object:
all_data[i] = pd.factorize(all_data[i])[0]
all_data = reduce_mem_usage(all_data)
 all_data.head()<prepare_x_and_y>
num_columns = ["sum_transaction", "num_transaction", "price", "member_duration_days", "no_of_seats"]
cat_columns = ["trip", "is_tx_promo", "airlines_name", "visited_city", "account_id", "service_class", "gender"]
X_train = all_data[all_data['winPlacePerc'].notnull() ].reset_index(drop=True)
 X_test = all_data[all_data['winPlacePerc'].isnull() ].drop(['winPlacePerc'], axis=1 ).reset_index(drop=True)
 del all_data
gc.collect()
Y_train = X_train.pop('winPlacePerc')
 X_test_grp = X_test[['matchId','groupId']].copy()
train_matchId = X_train['matchId']
X_train.drop(['matchId','groupId'], axis=1, inplace=True)
 X_test.drop(['matchId','groupId'], axis=1, inplace=True)
 print(X_train.shape, X_test.shape )<create_dataframe>
num_columns = ["avg_transaction", "price", "no_of_seats", "member_duration_days"]
cat_columns = ["trip", "is_tx_promo", "airlines_name", "visited_city", "account_id", "service_class", "gender", "num_transaction", "member_registered_quarter"]
def process_data2(df):
f = df.copy()
f["num_transaction"] = f.log_trx_proc.map(len)
 f["avg_transaction"] = f.log_trx_proc.map(sum)/ f["num_transaction"]
f["num_transaction"] = f["num_transaction"].astype(np.int32 ).map(lambda x: x if x <= 14 else 15)
 f.avg_transaction = f.avg_transaction / 10**4
f.price = f.price / 10**4
f["member_registered_quarter"] =(( f.member_duration_days - 329)// 90 ).astype("int32")
 f = f.drop(columns=["order_id", "hotel_id", "route", "log_transaction", "v_city_proc", "log_trx_proc", "is_cross_sell"])
 return f
print(pd.DataFrame([[val for val in dir() ], [sys.getsizeof(eval(val)) for val in dir() ]],
index=['name','size'] ).T.sort_values('size', ascending=False ).reset_index(drop=True)[:10])
<train_model>
def cast_columns(f):
for c in num_columns:
f[c] = f[c].astype(np.int32)
 for c in cat_columns:
f[c] = f[c].astype("category")
 return f
params={'learning_rate': 0.1,
'objective':'mae',
'metric':'mae',
'num_leaves': 31,
'verbose': 1,
'random_state':42,
'bagging_fraction': 0.7,
'feature_fraction': 0.7
}
reg = lgb.LGBMRegressor(**params, n_estimators=10000)
 reg.fit(X_train, Y_train)
 pred = reg.predict(X_test, num_iteration=reg.best_iteration_ )<concatenate>
ty = flight["hotel_id"].map(lambda x: 0 if x == "None" else 1 )
X_test_grp['_nofit.winPlacePerc'] = pred
group = X_test_grp.groupby(['matchId'])
 X_test_grp['winPlacePerc'] = pred
X_test_grp['_rank.winPlacePerc'] = group['winPlacePerc'].rank(method='min')
 X_test = pd.concat([X_test, X_test_grp], axis=1 )<feature_engineering>
train_split = flight.shape[0]
full_df = pd.concat(( flight, test), sort=True)
 p = cast_columns(process_data2(full_df))
tx, tex = p.iloc[:train_split], p.iloc[train_split:]
tx.shape, ty.shape, tex.shape
fullgroup =(X_test['numGroups'] == X_test['maxPlace'])
 subset = X_test.loc[fullgroup]
X_test.loc[fullgroup, 'winPlacePerc'] =(subset['_rank.winPlacePerc'].values - 1)/(subset['maxPlace'].values - 1)
 subset = X_test.loc[~fullgroup]
gap = 1.0 /(subset['maxPlace'].values - 1)
 new_perc = np.around(subset['winPlacePerc'].values / gap)* gap
X_test.loc[~fullgroup, 'winPlacePerc'] = new_perc
X_test['winPlacePerc'] = X_test['winPlacePerc'].clip(lower=0,upper=1 )<save_to_csv>
categ_feat_idx = np.where(tx.dtypes == "category")[0]
test = pd.read_csv('.. /input/test_V2.csv')
 test['matchId'] = test['matchId'].apply(lambda x: int(x,16))
test['groupId'] = test['groupId'].apply(lambda x: int(x,16))
submission = pd.merge(test, X_test[['matchId','groupId','winPlacePerc']])
 submission = submission[['Id','winPlacePerc']]
submission.to_csv("submission.csv", index=False )<set_options>
train_X, valid_X, train_y, valid_y = train_test_split(tx, ty, test_size=0.25, stratify=ty, random_state=17 )
%matplotlib inline<load_from_csv>
parameters = {
"random_seed": 17,
"eval_metric": "F1",
"custom_metric": ["F1", "Precision", "Recall"],
"loss_function": "Logloss",
"task_type": "GPU",
"random_strength": 3,
"od_type": "Iter",
"verbose":100,
"cat_features":categ_feat_idx
}
ctb = CatBoostClassifier(**parameters )
train_d = pd.read_csv(".. /input/train.csv")
 test_d = pd.read_csv(".. /input/test.csv")
 submission = pd.read_csv(".. /input/sample_submission.csv" )<train_model>
ctb.fit(train_X, train_y )
print("Train data size - rows:",train_d.shape[0]," columns:", train_d.shape[1])
 print("Test data size - rows:",test_d.shape[0]," columns:", test_d.shape[1])
 print("Submission data size - rows:",submission.shape[0]," columns:", submission.shape[1] )<count_missing_values>
get_metrics(valid_X,valid_y, e=ctb )
missingt1 = train_d.isnull().sum()
all_val = train_d.count()
missing_train_d = pd.concat([missingt1, all_val], axis=1, keys=['Missing', 'All'])
 missing_train_d<count_missing_values>
ctb.get_feature_importance(prettified=True )
missingt2 = test_d.isnull().sum()
all_val = test_d.count()
missing_test_d = pd.concat([missingt2, all_val], axis=1, keys=['Missing', 'All'])
 missing_test_d<count_unique_values>
!pip uninstall scikit-optimize --yes
!pip install git+https://github.com/darenr/scikit-optimize
train_d.nunique()<count_unique_values>
from scipy.stats import randint
from scipy.stats import uniform
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from sklearn.model_selection import cross_val_score
from sklearn.metrics import average_precision_score, roc_auc_score, mean_absolute_error, f1_score, make_scorer
from skopt import BayesSearchCV
from skopt import gp_minimize
from skopt.space import Real, Categorical, Integer
from skopt.utils import use_named_args
from skopt.callbacks import DeadlineStopper
from skopt.callbacks import VerboseCallback
from skopt.callbacks import DeltaXStopper
test_d.nunique()<concatenate>

 def report_perf(optimizer, X, y, title, callbacks=None):
start = time()
if callbacks:
optimizer.fit(X, y, callback=callbacks)
 else:
optimizer.fit(X, y)
 best_score = optimizer.best_score_
best_score_std = optimizer.cv_results_['std_test_score'][optimizer.best_index_]
best_params = optimizer.best_params_
print(( title + " took %.2f seconds, candidates checked: %d, best CV score: %.3f "
+u"\u00B1"+" %.3f")%(time() - start,
len(optimizer.cv_results_['params']),
best_score,
best_score_std))
print('Best parameters:')
 pprint.pprint(best_params)
 print()
return best_params
concatenated = pd.concat([train_d, test_d])
 concatenated.shape<count_unique_values>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=8)
 scorer = make_scorer(f1_score, average="macro")
 search_spaces = {'iterations': Integer(100, 1000),
'depth': Integer(4, 8),
'random_strength': Real(1e-9, 10, 'log-uniform'),
'bagging_temperature': Real(0.0, 1.0),
'border_count': Integer(128, 255),
'l2_leaf_reg': Integer(2, 30),
'scale_pos_weight':Real(1.0, 5.0, 'uniform')}
opt = BayesSearchCV(ctb,
search_spaces,
scoring=scorer,
cv=skf,
n_jobs=1,
return_train_score=False,
refit=True,
optimizer_kwargs={'base_estimator': 'GP'},
fit_params={'cat_features': categ_feat_idx, 'silent': True},
random_state=22,
verbose=True )
concatenated.nunique()<set_options>
best_params = report_perf(opt, tx, ty,'CatBoost',
callbacks=[VerboseCallback(100),
DeltaXStopper(0.0001),
DeadlineStopper(60*5)] )
warnings.filterwarnings("ignore" )<prepare_output>
best_params = {'bagging_temperature': 0.36426463210895726,
'border_count': 206,
'depth': 8,
'iterations': 342,
'l2_leaf_reg': 5,
'random_strength': 3.3149120967201682e-06,
'scale_pos_weight': 4.210304871687331}
temp = pd.DataFrame(train_d.landmark_id.value_counts().head(25))
temp.reset_index(inplace=True)
 temp.columns = ['landmark_id','count']
temp<create_dataframe>
tuned_model = CatBoostClassifier(**best_params, cat_features=categ_feat_idx, verbose=100, od_type="Iter" )
temp1 = pd.DataFrame(train_d.landmark_id.value_counts().tail(25))
temp1.reset_index(inplace=True)
 temp1.columns = ['landmark_id','count']
temp1<count_values>
train_y.value_counts() [0] // train_y.value_counts() [1]
freq_label = train_d['landmark_id'].value_counts() /train_d['landmark_id'].value_counts().sum()<save_to_csv>
ctb2 = CatBoostClassifier(**{"random_seed": 40,
"eval_metric": "F1",
"custom_metric": ["F1", "Precision", "Recall"],
"loss_function": "Logloss",
"task_type": "GPU",
"random_strength": 3,
"od_type": "Iter",
"scale_pos_weight": 12,
"verbose":100,
"cat_features":categ_feat_idx} )
submission['landmarks'] = '%d %2.2f' %(freq_label.index[0], freq_label.values[0])
 submission.to_csv('submission.csv', index=False )<randomize_order>
flight2 = flight.copy()
d = flight.duplicated(subset=("account_id", "airlines_name", "trip", "is_tx_promo", "price"))
flight2 = flight[~d]
np.random.seed(2018)
 r_idx = lambda : np.random.choice(freq_label.index, p = freq_label.values )<save_to_csv>
ty2 = flight2.hotel_id.map(lambda x: 0 if x == "None" else 1)
 tx2 = cast_columns(process_data2(flight2)) [tx.columns]
r_score = lambda idx: '%d %2.4f' %(freq_label.index[idx], freq_label.values[idx])
 submission['landmarks'] = submission.id.map(lambda _: r_score(r_idx()))
submission.to_csv('rand_submission.csv', index=False )<feature_engineering>
categ_feat_idx2 = np.where(tx2.dtypes == "category")[0]
temp_l1 = list()
for path in train_d['url']:
temp_l1.append(( path.split('//', 1)[1] ).split('/', 1)[0])
 train_d['site_name'] = temp_l1
temp_l1 = list()
for path in test_d['url']:
temp_l1.append(( path.split('//', 1)[1] ).split('/', 1)[0])
 test_d['site_name'] = temp_l1
print("Training data size",train_d.shape)
 print("test data size",test_d.shape )<define_variables>
ctb4 = CatBoostClassifier(**{"random_seed": 40,
"eval_metric": "F1",
"custom_metric": ["F1", "Precision", "Recall"],
"loss_function": "Logloss",
"task_type": "GPU",
"random_strength": 3,
"od_type": "Iter",
"scale_pos_weight": 3.5,
"verbose":100,
"cat_features":categ_feat_idx2} )
train_d1 = train_d
train_d1.head()<drop_column>
train_X2, valid_X2, train_y2, valid_y2 = train_test_split(tx2, ty2, test_size=0.25, stratify=ty2, random_state=17 )
train_d1 = train_d1.drop('url',1)
 train_d1.head()<create_dataframe>
temp = pd.DataFrame(train_d1.site_name.value_counts())
 temp.reset_index(inplace=True)
 temp.columns = ['site_name','count']
temp<drop_column>
tuned_model.fit(tx, ty )
test_d1 = test_d
test_d1.head()<drop_column>
ctb.fit(tx, ty)
 ctb2.fit(tx, ty)
 ctb4.fit(tx2, ty2 )
test_d1 = test_d1.drop('url',1)
 test_d1.head()<create_dataframe>
get_metrics(valid_X,valid_y, e=vc )
temp = pd.DataFrame(test_d.site_name.value_counts())
 temp.reset_index(inplace=True)
 temp.columns = ['site_name','count']
temp<prepare_output>
eclf = EnsembleVoteClassifier(clfs=[ctb2, tuned_model], weights=[1,1.1], refit=False, voting="soft")
 eclf.fit(tx, ty )
train_d2 = train_d
test_d2 = test_d
train_d2.head()<drop_column>
f_pred = pd.Series(eclf.predict(tex.to_numpy())).map(lambda x: "no" if x == 0 else "yes")
 f_pred.value_counts()
train_d2 = train_d2.drop('site_name',1)
 train_d2.head()<drop_column>
submission = test[["order_id"]]
submission["is_cross_sell"] = f_pred
submission
<merge><EOS> 
submission.to_csv('submission.csv', header=True, index=False )