Kaggle competition: Google QUEST Q&A Labeling
@jit
def haversine_plus(lon1, lat1, lon2, lat2):

 lon1 = np.radians(lon1)
 lat1 = np.radians(lat1)
 lon2 = np.radians(lon2)
 lat2 = np.radians(lat2)
 dlon = np.subtract(lon2, lon1)
 dlat = np.subtract(lat2, lat1)
 a = np.add(np.power(np.sin(np.divide(dlat, 2)) , 2),
np.multiply(np.cos(lat1),
np.multiply(np.cos(lat2),
np.power(np.sin(np.divide(dlon, 2)) , 2))))
haversine = np.multiply(2, np.arcsin(np.sqrt(a)))
 return {
'haversine': haversine,
'latlon1': np.subtract(np.multiply(lon1, lat1), np.multiply(lon2, lat2)) ,
}
@jit
def process_flux(df):
flux_ratio_sq = np.power(df['flux'].values / df['flux_err'].values, 2.0)
 df_flux = pd.DataFrame({
'flux_ratio_sq': flux_ratio_sq,
'flux_by_flux_ratio_sq': df['flux'].values * flux_ratio_sq
},
index=df.index 
)
 return pd.concat([df, df_flux], axis=1)
 @jit
def process_flux_agg(df):
flux_w_mean = df['flux_by_flux_ratio_sq_sum'].values / df['flux_ratio_sq_sum'].values
flux_diff = df['flux_max'].values - df['flux_min'].values
df_flux_agg = pd.DataFrame({
'flux_w_mean': flux_w_mean,
'flux_diff1': flux_diff,
'flux_diff2': flux_diff / df['flux_mean'].values,
'flux_diff3': flux_diff / flux_w_mean,
},
index=df.index 
)
 return pd.concat([df, df_flux_agg], axis=1)
 AGGS = {
'flux': ['min', 'max', 'mean', 'median', 'std', 'skew'],
'flux_err': ['min', 'max', 'mean', 'median', 'std', 'skew'],
'detected': ['min', 'max', 'mean', 'median', 'std', 'skew'],
'flux_ratio_sq': ['sum', 'skew'],
'flux_by_flux_ratio_sq': ['sum', 'skew'],
}
FCP = {
'flux': {
'longest_strike_above_mean': None,
'longest_strike_below_mean': None,
'mean_change': None,
'mean_abs_change': None,
'length': None,
},
'flux_by_flux_ratio_sq': {
'longest_strike_above_mean': None,
'longest_strike_below_mean': None,
},
'flux_passband': {
'fft_coefficient': [
{'coeff': 0, 'attr': 'abs'},
{'coeff': 1, 'attr': 'abs'}
],
'kurtosis': None,
'skewness': None,
},
'mjd': {
'maximum': None,
'minimum': None,
'mean_change': None,
'mean_abs_change': None,
},
}
def featurize(df, df_meta, n_jobs=4):

 df = process_flux(df)
 aggs = AGGS
fcp = FCP
agg_df = df.groupby('object_id' ).agg(aggs)
 agg_df.columns = ['{}_{}'.format(k, agg)for k in aggs.keys() for agg in aggs[k]]
agg_df = process_flux_agg(agg_df)
 agg_df_ts_flux_passband = extract_features(df,
column_id='object_id',
column_sort='mjd',
column_kind='passband',
column_value='flux',
default_fc_parameters=fcp['flux_passband'],
n_jobs=n_jobs 
)
 agg_df_ts_flux = extract_features(df,
column_id='object_id',
column_value='flux',
default_fc_parameters=fcp['flux'], n_jobs=n_jobs)
 agg_df_ts_flux_by_flux_ratio_sq = extract_features(df,
column_id='object_id',
column_value='flux_by_flux_ratio_sq',
default_fc_parameters=fcp['flux_by_flux_ratio_sq'],
n_jobs=n_jobs)
 df_det = df[df['detected'] == 1].copy()
agg_df_mjd = extract_features(df_det,
column_id='object_id',
column_value='mjd',
default_fc_parameters=fcp['mjd'], n_jobs=n_jobs)
 agg_df_mjd['mjd_diff_det'] = agg_df_mjd['mjd__maximum'].values - agg_df_mjd['mjd__minimum'].values
del agg_df_mjd['mjd__maximum'], agg_df_mjd['mjd__minimum']
agg_df_ts_flux_passband.index.rename('object_id', inplace=True)
 agg_df_ts_flux.index.rename('object_id', inplace=True)
 agg_df_ts_flux_by_flux_ratio_sq.index.rename('object_id', inplace=True)
 agg_df_mjd.index.rename('object_id', inplace=True)
 agg_df_ts = pd.concat([agg_df,
agg_df_ts_flux_passband,
agg_df_ts_flux,
agg_df_ts_flux_by_flux_ratio_sq,
agg_df_mjd], axis=1 ).reset_index()
result = agg_df_ts.merge(right=df_meta, how='left', on='object_id')
 return result
def process_meta(filename):
meta_df = pd.read_csv(filename)
 meta_dict = dict()
meta_dict.update(haversine_plus(meta_df['ra'].values, meta_df['decl'].values, meta_df['gal_l'].values, meta_df['gal_b'].values))
meta_dict['hostgal_photoz_certain'] = np.multiply(
 meta_df['hostgal_photoz'].values,
np.exp(meta_df['hostgal_photoz_err'].values))
meta_df = pd.concat([meta_df, pd.DataFrame(meta_dict, index=meta_df.index)], axis=1)
 return meta_df
def multi_weighted_logloss(y_true, y_preds, classes, class_weights):

 y_p = y_preds.reshape(y_true.shape[0], len(classes), order='F')
 y_ohe = pd.get_dummies(y_true)
 y_p = np.clip(a=y_p, a_min=1e-15, a_max=1 - 1e-15)
 y_p_log = np.log(y_p)
 y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)
 nb_pos = y_ohe.sum(axis=0 ).values.astype(float)
 class_arr = np.array([class_weights[k] for k in sorted(class_weights.keys())])
 y_w = y_log_ones * class_arr / nb_pos
loss = - np.sum(y_w)/ np.sum(class_arr)
 return loss
def lgbm_multi_weighted_logloss(y_true, y_preds):

 galactic_classes = [6, 16, 53, 65, 92]
extragalactic_classes = [15, 42, 52, 62, 64, 67, 88, 90, 95]
galactic_classes_weights = {c: 1 for c in galactic_classes}
extragalactic_classes_weights = {c: 1 for c in extragalactic_classes}
extragalactic_classes_weights.update({c: 2 for c in [64, 15]})
 if len(y_preds)== 5 * y_true.shape[0]:
classes = galactic_classes
class_weights = galactic_classes_weights
else:
classes = extragalactic_classes
class_weights = extragalactic_classes_weights
loss = multi_weighted_logloss(y_true, y_preds, classes, class_weights)
 return 'wloss', loss, False
def xgb_multi_weighted_logloss(y_predicted, y_true, classes, class_weights):
loss = multi_weighted_logloss(y_true.get_label() , y_predicted, classes, class_weights)
 return 'wloss', loss<categorify>
n_original = len(data_table)
 score_list = []
for i in range(n_original):
print('=========================')
 print(data_table[i]['Name'])
 print('=========================')
 data_table[i]['valid'] = pd.DataFrame(data_table[i]['valid'] ).rank() / len(data_table[i]['valid'])
 data_table[i]['test_preds'] = pd.DataFrame(data_table[i]['test_preds'] ).rank() / len(data_table[i]['test_preds'])
 data_table.append(copy.deepcopy(data_table[i]))
score1 = compute_spearmanr_ignore_nan(np.asarray(train[targets].copy()), np.asarray(data_table[i]['valid'].copy()))
optimize_preds(train, data_table[i]['valid'], data_table[i]['test_preds'])
 score2 = compute_spearmanr_ignore_nan(np.asarray(train[targets].copy()), np.asarray(data_table[i]['valid'].copy()))
score_list.append([score1, score2] )
galactic_classes = [6, 16, 53, 65, 92]
extragalactic_classes = [15, 42, 52, 62, 64, 67, 88, 90, 95]
def predict_chunk(df_, clfs_gal_, clfs_ext_, meta_, features):
full_test = featurize(df_, meta_)
 full_test.fillna(0, inplace=True)
 galactic_cut = full_test['hostgal_photoz'] == 0
gal_test = full_test[galactic_cut]
ext_test = full_test[~galactic_cut]
preds_gal = None
if not gal_test.empty:
for clf in clfs_gal_:
if preds_gal is None:
preds_gal = clf.predict_proba(gal_test[features])
 else:
preds_gal += clf.predict_proba(gal_test[features])
 preds_gal = preds_gal / len(clfs_gal_)
 preds_99_gal = np.ones(preds_gal.shape[0])
 for i in range(preds_gal.shape[1]):
preds_99_gal *=(1 - preds_gal[:, i])
 preds_gal = pd.DataFrame(preds_gal,
columns=['class_{}'.format(s)for s in clfs_gal_[0].classes_])
 assert preds_gal.shape[0] == gal_test.shape[0], 'len of preds={}, test={}'.format(preds_gal.shape[0], gal_test.shape[0])
 preds_gal['object_id'] = gal_test['object_id'].values
for c in ['class_{}'.format(s)for s in extragalactic_classes]:
preds_gal.insert(0, c, 0.0)
 preds_gal['class_99'] = 0.017 * preds_99_gal / np.mean(preds_99_gal)
 preds_ext = None
if not ext_test.empty:
for clf in clfs_ext_:
if preds_ext is None:
preds_ext = clf.predict_proba(ext_test[features])
 else:
preds_ext += clf.predict_proba(ext_test[features])
 preds_ext = preds_ext / len(clfs_ext_)
 preds_99_ext = np.ones(preds_ext.shape[0])
 for i in range(preds_ext.shape[1]):
preds_99_ext *=(1 - preds_ext[:, i])
 preds_ext = pd.DataFrame(preds_ext,
columns=['class_{}'.format(s)for s in clfs_ext_[0].classes_])
 assert preds_ext.shape[0] == ext_test.shape[0], 'len of preds={}, test={}'.format(preds_ext.shape[0], ext_test.shape[0])
 preds_ext['object_id'] = ext_test['object_id'].values
for c in ['class_{}'.format(s)for s in galactic_classes]:
preds_ext.insert(0, c, 0.0)
 preds_ext['class_99'] = 0.17 * preds_99_ext / np.mean(preds_99_ext)
 preds_df_ = pd.concat([preds_gal, preds_ext], ignore_index=True, sort=False)
 return preds_df_
def process_test(clfs_gal, clfs_ext,
features,
filename='predictions.csv',
chunks=5000000):
start = time.time()
meta_test = process_meta('.. /input/test_set_metadata.csv')
 remain_df = None
for i_c, df in enumerate(pd.read_csv('.. /input/test_set.csv', chunksize=chunks, iterator=True)) :
unique_ids = np.unique(df['object_id'])
 new_remain_df = df.loc[df['object_id'] == unique_ids[-1]].copy()
if remain_df is None:
df = df.loc[df['object_id'].isin(unique_ids[:-1])]
else:
df = pd.concat([remain_df, df.loc[df['object_id'].isin(unique_ids[:-1])]], axis=0)
 remain_df = new_remain_df
preds_df = predict_chunk(df_=df,
clfs_gal_=clfs_gal,
clfs_ext_=clfs_ext,
meta_=meta_test,
features=features)
 if i_c == 0:
preds_df.to_csv(filename, header=True, mode='a', index=False)
 else:
preds_df.to_csv(filename, header=False, mode='a', index=False)
 del preds_df
gc.collect()
print('{:15d} done in {:5.1f} minutes'.format(
 chunks *(i_c + 1),(time.time() - start)/ 60), flush=True)
 preds_df = predict_chunk(df_=remain_df,
clfs_gal_=clfs_gal,
clfs_ext_=clfs_ext,
meta_=meta_test,
features=features)
 preds_df.to_csv(filename, header=False, mode='a', index=False)
 return
def lgbm_modeling_cross_validation(params,
full_train,
y,
classes,
class_weights,
id,
part,
nr_fold=5,
random_state=99, 
):
w = y.value_counts()
weights = {i: np.sum(w)/ w[i] for i in w.index}
clfs = []
importances = pd.DataFrame()
folds = StratifiedKFold(n_splits=nr_fold,
shuffle=True,
random_state=random_state)
 oof_preds = np.zeros(( len(full_train), np.unique(y ).shape[0]))
for fold_,(trn_, val_)in enumerate(folds.split(y, y)) :
trn_x, trn_y = full_train.iloc[trn_], y.iloc[trn_]
val_x, val_y = full_train.iloc[val_], y.iloc[val_]
clf = LGBMClassifier(**params)
 clf.fit(
 trn_x, trn_y,
eval_set=[(trn_x, trn_y),(val_x, val_y)],
eval_metric=lgbm_multi_weighted_logloss,
verbose=100,
early_stopping_rounds=50,
sample_weight=trn_y.map(weights)
)
 clfs.append(clf)
 oof_preds[val_, :] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)
 print('no {}-fold loss: {}'.format(fold_ + 1,
multi_weighted_logloss(val_y, oof_preds[val_, :],
classes, class_weights)))
 imp_df = pd.DataFrame({
'feature': full_train.columns,
'gain': clf.feature_importances_,
'fold': [fold_ + 1] * len(full_train.columns),
})
 importances = pd.concat([importances, imp_df], axis=0, sort=False)
 score = multi_weighted_logloss(y_true=y, y_preds=oof_preds,
classes=classes, class_weights=class_weights)
 print('MULTI WEIGHTED LOG LOSS: {:.5f}'.format(score))
oof_preds_pd = pd.DataFrame(data=oof_preds, columns=['class_{}'.format(s)for s in classes])
 pd.concat([id, oof_preds_pd], axis=1 ).to_csv('lgbm_train_oof_preds_{}.csv'.format(part), index=False)
 df_importances = save_importances(importances_=importances)
 df_importances.to_csv('lgbm_importances_{}.csv'.format(part), index=False)
 return clfs, score
def save_importances(importances_):
mean_gain = importances_[['gain', 'feature']].groupby('feature' ).mean()
importances_['mean_gain'] = importances_['feature'].map(mean_gain['gain'])
 return importances_<init_hyperparams>
for i in range(len(score_list)) :
score1, score2 = score_list[i][0], score_list[i][1]
print('{:.6f} -> {:.6f}({:.6f}): {}'.format(score1, score2, score2 - score1, data_table[i]['Name']))
xgb_params = {
'objective': 'multi:softprob',
'eval_metric': 'mlogloss',
'silent': True,
'num_class': 14,
'booster': 'gbtree',
'n_jobs': 4,
'n_estimators': 1000,
'tree_method': 'hist',
'grow_policy': 'lossguide',
'base_score': 0.25,
'max_depth': 7,
'max_delta_step': 2,
'learning_rate': 0.03,
'max_leaves': 11,
'min_child_weight': 64,
'gamma': 0.1,
'subsample': 0.7,
'colsample_bytree': 0.68,
'reg_alpha': 0.01,
'reg_lambda': 10.,
'seed': 1234
}
lgbm_params = {
'device': 'cpu',
'objective': 'multiclass',
'num_class': 14,
'boosting_type': 'gbdt',
'n_jobs': -1,
'max_depth': 7,
'n_estimators': 500,
'subsample_freq': 2,
'subsample_for_bin': 5000,
'min_data_per_group': 100,
'max_cat_to_onehot': 4,
'cat_l2': 1.0,
'cat_smooth': 59.5,
'max_cat_threshold': 32,
'metric_freq': 10,
'verbosity': -1,
'metric': 'multi_logloss',
'xgboost_dart_mode': False,
'uniform_drop': False,
'colsample_bytree': 0.5,
'drop_rate': 0.173,
'learning_rate': 0.0267,
'max_drop': 5,
'min_child_samples': 10,
'min_child_weight': 100.0,
'min_split_gain': 0.1,
'num_leaves': 7,
'reg_alpha': 0.1,
'reg_lambda': 0.00023,
'skip_drop': 0.44,
'subsample': 0.75
}<prepare_x_and_y>
validation_totale = train[targets].copy()
test_preds_totale = sample.copy()
def train_model(full_train, classes, class_weights, part):
if 'target' in full_train:
y = full_train['target']
del full_train['target']
if 'object_id' in full_train:
object_id = full_train['object_id']
del full_train['object_id']
del full_train['hostgal_specz']
del full_train['ra'], full_train['decl'], full_train['gal_l'], full_train['gal_b']
del full_train['ddf']
eval_func = partial(lgbm_modeling_cross_validation,
full_train=full_train,
y=y,
classes=classes,
class_weights=class_weights,
id=object_id,
part=part,
nr_fold=5,
random_state=99, 
)
 lgbm_params.update({'n_estimators': 2000})
 clfs, score = eval_func(lgbm_params)
 return clfs, score<load_from_csv>
for title in targets:
y1 = np.asarray(train[title].copy())
 X_list = [ np.asarray(x['valid'][title].copy())for x in data_table ]
test_list = [ np.asarray(x['test_preds'][title].copy())for x in data_table ]
corr_list = [ spearmanr(y1, x ).correlation for x in X_list ]
i = np.argmax(corr_list)
 corr_best = copy.deepcopy(corr_list[i])
 x_best = copy.deepcopy(X_list[i])
 test_best = copy.deepcopy(test_list[i])
 for weight_range in np.arange(0.45, 0.95, 0.005):
corr_list2 = [c*c for c in corr_list]
cmin = np.min(corr_list2)
 cmax = np.max(corr_list2)
 weights =(corr_list2 - cmin)/(cmax - cmin)
 weights *= weight_range
weights +=(1.0 - weight_range)
 x = np.average(X_list, axis=0, weights=weights)
 x = rankdata(x)/ len(x)
 corr = spearmanr(y1, x ).correlation
if corr > corr_best:
corr_best = copy.deepcopy(corr)
 x_best = copy.deepcopy(x)
 t = np.average(test_list, axis=0, weights=weights)
 t = rankdata(t)/ len(t)
 test_best = copy.deepcopy(t)
 validation_totale[title] = copy.deepcopy(x_best)
 test_preds_totale[title] = copy.deepcopy(test_best )
def main(argc, argv):
meta_train = process_meta('.. /input/training_set_metadata.csv')
 train = pd.read_csv('.. /input/training_set.csv')
 full_train = featurize(train, meta_train)
 galactic_classes_weights = {c: 1 for c in galactic_classes}
extragalactic_classes_weights = {c: 1 for c in extragalactic_classes}
extragalactic_classes_weights.update({c: 2 for c in [64, 15]})
 full_train.fillna(0, inplace=True)
 galactic_cut = full_train['hostgal_photoz'] == 0
clfs_gal, score_gal = train_model(full_train[galactic_cut],
galactic_classes, galactic_classes_weights, 'gal')
 clfs_ext, score_ext = train_model(full_train[~galactic_cut],
extragalactic_classes, extragalactic_classes_weights, 'ext')
 filename = 'subm_{:.6f}_{:.6f}_{}.csv'.format(
 score_gal, score_ext,
datetime.now().strftime('%Y-%m-%d-%H-%M'))
print('save to {}'.format(filename))
if 'target' in full_train:
y = full_train['target']
del full_train['target']
if 'object_id' in full_train:
object_id = full_train['object_id']
del full_train['object_id']
del full_train['hostgal_specz']
del full_train['ra'], full_train['decl'], full_train['gal_l'], full_train['gal_b']
del full_train['ddf']
process_test(clfs_gal, clfs_ext,
features=full_train.columns,
filename=filename,
chunks=5000000)
 z = pd.read_csv(filename)
 print("Shape BEFORE grouping: {}".format(z.shape))
z = z.groupby('object_id' ).mean()
print("Shape AFTER grouping: {}".format(z.shape))
z.to_csv('single_{}'.format(filename), index=True )<load_from_csv>
compute_spearmanr_ignore_nan(np.asarray(train[targets].copy()), np.asarray(validation_totale[targets].copy()))
pdf=pd.read_csv('.. /input/forked-dart-w-ideas-from-kernels-added-smote/single_subm_0.648970_2018-11-21-09-03.csv')
 print(pdf.shape)
 pdf.head() 
<prepare_x_and_y>
def function_spelling(row):
if row in 'CULTURE':
return 1
else:
return 0
validation_totale['question_type_spelling'] = train['category'].apply(function_spelling)*validation_totale['question_type_spelling']
def GenUnknown(data):
return(((((( data["mymedian"])+(((data["mymean"])/ 2.0)))/2.0)) +(((((1.0)-(((data["mymax"])*(((data["mymax"])*(data["mymax"])))))))/ 2.0)))/2.0)
 feats = ['class_6', 'class_15', 'class_16', 'class_42', 'class_52', 'class_53',
'class_62', 'class_64', 'class_65', 'class_67', 'class_88', 'class_90',
'class_92', 'class_95']
y = pd.DataFrame()
y['mymean'] = pdf[feats].mean(axis=1)
 y['mymedian'] = pdf[feats].median(axis=1)
 y['mymax'] = pdf[feats].max(axis=1)
 pdf['class_99'] = GenUnknown(y)
<save_to_csv>
compute_spearmanr_ignore_nan(np.asarray(train[targets].copy()), np.asarray(validation_totale[targets].copy()))
pdf.to_csv('improvedClass99forSmoteDart.csv', index=False )<set_options>
submission = pd.read_csv(path_join(data_dir, 'sample_submission.csv'))
for title in targets:
for i, row in test_preds_totale.iterrows() :
submission.loc[submission['qa_id'] == row['qa_id'], title] = row[title]
np.warnings.filterwarnings('ignore')
<compute_train_metric>
submission['question_type_spelling'] = test['category'].apply(function_spelling)*submission['question_type_spelling']
def multi_weighted_logloss(y_true, y_preds, classes, class_weights):

 y_p = y_preds.reshape(y_true.shape[0], len(classes), order='F')
 y_ohe = pd.get_dummies(y_true%100)
 y_p = np.clip(a=y_p, a_min=1e-15, a_max=1 - 1e-15)
 y_p_log = np.log(y_p)
 y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)
 nb_pos = y_ohe.sum(axis=0 ).values.astype(float)
 class_arr = np.array([class_weights[k] for k in sorted(class_weights.keys())])
 y_w = y_log_ones * class_arr / nb_pos
loss = - np.sum(y_w)/ np.sum(class_arr)
 return loss<compute_test_metric>
compute_spearmanr_ignore_nan(np.asarray(train[targets].copy()), np.asarray(validation_totale[targets].copy()))
def lgbm_multi_weighted_logloss(y_true, y_preds):

 y_true=y_true%100
y_preds=y_preds%100
classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]
class_weights = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}
loss = multi_weighted_logloss(y_true, y_preds, classes, class_weights)
 return 'wloss', loss, False
def xgb_multi_weighted_logloss(y_predicted, y_true, classes, class_weights):
loss = multi_weighted_logloss(y_true.get_label() , y_predicted,
classes, class_weights)
 return 'wloss', loss<compute_train_metric>
for title in targets:
submission[title] = submission[title].apply(lambda x: 1 if x>1 else x )
<train_model><EOS> 
submission.to_csv("submission.csv", index = False)
 submission.head()