Kaggle competition: PLAsTiCC Astronomical Classification
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 train_columns = scaled_train_X.columns.values<init_hyperparams>

 np.warnings.filterwarnings('ignore')
params = {'num_leaves': 35,
'min_data_in_leaf': 10,
'min_sum_hessian_in_leaf': 10.0,
'min_child_samples':10,
'boost_from_average':'false',
'objective':'regression',
'max_depth': -1,
'learning_rate': 0.001,
"boosting": "gbdt",
"feature_fraction": 0.91,
"bagging_freq": 1,
"bagging_fraction": 0.2,
"bagging_seed": 42,
"metric": 'mae',
'reg_alpha': 0.4002,
'reg_lambda':0.8003,
"verbosity": -1,
"nthread": -1,
"random_state": 42}<train_model>
def multi_weighted_logloss(y_true, y_preds, classes, class_weights):

 y_p = y_preds.reshape(y_true.shape[0], len(classes), order='F')
 y_ohe = pd.get_dummies(y_true)
 y_p = np.clip(a=y_p, a_min=1e-15, a_max=1 - 1e-15)
 y_p_log = np.log(y_p)
 y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)
 nb_pos = y_ohe.sum(axis=0 ).values.astype(float)
 class_arr = np.array([class_weights[k] for k in sorted(class_weights.keys())])
 y_w = y_log_ones * class_arr / nb_pos
loss = - np.sum(y_w)/ np.sum(class_arr)
 return loss
oof = np.zeros(len(scaled_train_X))
predictions = np.zeros(len(scaled_test_X))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(scaled_train_X,train_y.values)) :
strLog = "fold {}".format(fold_)
 print(strLog)
 X_tr, X_val = scaled_train_X.iloc[trn_idx], scaled_train_X.iloc[val_idx]
y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]
model = lgb.LGBMRegressor(**params, n_estimators = 20000, n_jobs = -1)
 model.fit(X_tr,
y_tr,
eval_set=[(X_tr, y_tr),(X_val, y_val)],
eval_metric='mae',
verbose=1000,
early_stopping_rounds=500)
 oof[val_idx] = model.predict(X_val, num_iteration=model.best_iteration_)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = train_columns
fold_importance_df["importance"] = model.feature_importances_[:len(train_columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += model.predict(scaled_test_X, num_iteration=model.best_iteration_)/ folds.n_splits
print("Overall CV score: {:<8.5f}".format(mean_absolute_error(train_y.values, oof)) )<save_to_csv>
def lgbm_multi_weighted_logloss(y_true, y_preds):

 classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]
class_weights = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}
loss = multi_weighted_logloss(y_true, y_preds, classes, class_weights)
 return 'wloss', loss, False
def xgb_multi_weighted_logloss(y_predicted, y_true, classes, class_weights):
loss = multi_weighted_logloss(y_true.get_label() , y_predicted,
classes, class_weights)
 return 'wloss', loss
submission.time_to_failure = predictions
submission.to_csv('submission.csv',index=True )<set_options>
def save_importances(importances_):
mean_gain = importances_[['gain', 'feature']].groupby('feature' ).mean()
importances_['mean_gain'] = importances_['feature'].map(mean_gain['gain'])
 return importances_
warnings.filterwarnings("ignore")
 warnings.simplefilter(action='ignore', category=FutureWarning)
 pd.options.display.precision = 15<import_modules>
def smoteAdataset(Xig_train, yig_train, Xig_test, yig_test):
sm=SMOTE(random_state=2)
 Xig_train_res, yig_train_res = sm.fit_sample(Xig_train, yig_train.ravel())
 return Xig_train_res, pd.Series(yig_train_res), Xig_test, pd.Series(yig_test )
import os
import gc
from joblib import Parallel, delayed<feature_engineering>
def lgbm_modeling_cross_validation(params,
full_train,
y,
classes,
class_weights,
nr_fold=12,
random_state=1):
w = y.value_counts()
weights = {i : np.sum(w)/ w[i] for i in w.index}
clfs = []
importances = pd.DataFrame()
folds = StratifiedKFold(n_splits=nr_fold,
shuffle=True,
random_state=random_state)
 oof_preds = np.zeros(( len(full_train), np.unique(y ).shape[0]))
for fold_,(trn_, val_)in enumerate(folds.split(y, y)) :
trn_x, trn_y = full_train.iloc[trn_], y.iloc[trn_]
val_x, val_y = full_train.iloc[val_], y.iloc[val_]
trn_xa, trn_y, val_xa, val_y=smoteAdataset(trn_x.values, trn_y.values, val_x.values, val_y.values)
 trn_x=pd.DataFrame(data=trn_xa, columns=trn_x.columns)
 val_x=pd.DataFrame(data=val_xa, columns=val_x.columns)
 clf = LGBMClassifier(**params)
 clf.fit(
 trn_x, trn_y,
eval_set=[(trn_x, trn_y),(val_x, val_y)],
eval_metric=lgbm_multi_weighted_logloss,
verbose=100,
early_stopping_rounds=50,
sample_weight=trn_y.map(weights)
)
 clfs.append(clf)
 oof_preds[val_, :] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)
 print('no {}-fold loss: {}'.format(fold_ + 1,
multi_weighted_logloss(val_y, oof_preds[val_, :],
classes, class_weights)))
 imp_df = pd.DataFrame({
'feature': full_train.columns,
'gain': clf.feature_importances_,
'fold': [fold_ + 1] * len(full_train.columns),
})
 importances = pd.concat([importances, imp_df], axis=0, sort=False)
 score = multi_weighted_logloss(y_true=y, y_preds=oof_preds,
classes=classes, class_weights=class_weights)
 print('MULTI WEIGHTED LOG LOSS: {:.5f}'.format(score))
df_importances = save_importances(importances_=importances)
 df_importances.to_csv('lgbm_importances.csv', index=False)
 return clfs, score
def classic_sta_lta(x, Ns, Nl):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
lta[Nl:-Ns] = lta[Nl:-Ns] - lta[:-Nl-Ns]
lta /= Nl
sta[Nl+Ns-1:] = sta[Nl+Ns-1:] - sta[Nl-1:-Ns]
sta /= Ns
sta[:Nl - 1 + Ns] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta<load_from_csv>
def predict_chunk(df_, clfs_, features, train_mean):
agg_ = df_
full_test = agg_.reset_index()
full_test = full_test.fillna(0)
 full_test = full_test.round(5)
 preds_ = None
for clf in clfs_:
if preds_ is None:
preds_ = clf.predict_proba(full_test[features])/ len(clfs_)
 else:
preds_ += clf.predict_proba(full_test[features])/ len(clfs_)
 preds_99 = np.ones(preds_.shape[0])
 for i in range(preds_.shape[1]):
preds_99 *=(1 - preds_[:, i])
 preds_df_ = pd.DataFrame(preds_, columns=['class_' + str(s)for s in clfs_[0].classes_])
 preds_df_['object_id'] = full_test['object_id']
preds_df_['class_99'] = 0.14 * preds_99 / np.mean(preds_99)
 return preds_df_
class FeatureGenerator(object):
def __init__(self, dtype, n_jobs=1, chunk_size=None):
self.chunk_size = chunk_size
self.dtype = dtype
self.filename = None
self.n_jobs = n_jobs
self.test_files = []
if self.dtype == 'train':
self.filename = '.. /input/train.csv'
self.total_data = int(629145481 / self.chunk_size)
 else:
submission = pd.read_csv('.. /input/sample_submission.csv')
 for seg_id in submission.seg_id.values:
self.test_files.append(( seg_id, '.. /input/test/' + seg_id + '.csv'))
self.total_data = int(len(submission))
def read_chunks(self):
if self.dtype == 'train':
iter_df = pd.read_csv(self.filename, iterator=True, chunksize=self.chunk_size,
dtype={'acoustic_data': np.float64, 'time_to_failure': np.float64})
 for counter, df in enumerate(iter_df):
x = df.acoustic_data.values
y = df.time_to_failure.values[-1]
seg_id = 'train_' + str(counter)
 del df
yield seg_id, x, y
else:
for seg_id, f in self.test_files:
df = pd.read_csv(f, dtype={'acoustic_data': np.float64})
 x = df.acoustic_data.values[-self.chunk_size:]
del df
yield seg_id, x, -999
def features(self, x, y, seg_id):
feature_dict = dict()
feature_dict['target'] = y
feature_dict['seg_id'] = seg_id
feature_dict['mean'] = np.mean(x)
 feature_dict['max'] = np.max(x)
 feature_dict['min'] = np.min(x)
 feature_dict['std'] = np.std(x)
 feature_dict['var'] = np.var(x)
 feature_dict['quantile_03'] = np.quantile(x, 0.03)
 feature_dict['skew'] = sp.stats.skew(x)
 feature_dict['kurtosis'] = sp.stats.kurtosis(x)
 feature_dict['moment_3'] = sp.stats.moment(x, 3)
 pct_change = pd.Series(x ).pct_change()
pct_change[pct_change == -inf] = 0
pct_change[pct_change == inf] = 0
feature_dict['pct_change_mean'] = pct_change.mean()
rate_change = pd.Series(x ).pct_change().pct_change()
rate_change[rate_change == -inf] = 0
rate_change[rate_change == inf] = 0
feature_dict['rate_change_max'] = rate_change.max()
feature_dict['rate_change_mean'] = rate_change.mean()
feature_dict['classic_sta_lta_mean'] = classic_sta_lta(x, 100, 5000 ).mean()
window_size = 10
x_roll_std = pd.Series(x ).rolling(window_size ).std().dropna().values
feature_dict['q03_roll_std_' + str(window_size)] = np.quantile(x_roll_std, 0.03)
 window_size = 150
x_roll_std = pd.Series(x ).rolling(window_size ).std().dropna().values
feature_dict['q03_roll_std_' + str(window_size)] = np.quantile(x_roll_std, 0.03)
 return feature_dict
def generate(self):
feature_list = []
res = Parallel(n_jobs=self.n_jobs,
backend='threading' )(delayed(self.features )(x, y, s)
 for s, x, y in tqdm(self.read_chunks() , total=self.total_data))
for r in res:
feature_list.append(r)
 return pd.DataFrame(feature_list)
 training_fg = FeatureGenerator(dtype='train', n_jobs=10, chunk_size=150000)
 training_data = training_fg.generate()
test_fg = FeatureGenerator(dtype='test', n_jobs=10, chunk_size=150000)
 test_data = test_fg.generate() 
<prepare_x_and_y>
def process_test(clfs,
testdf,
full_train,
train_mean,
filename='submission.csv',
chunks=40615):
start = time.time()
chunks = 40615
testdf=testdf.round(5)
 testdf.to_csv(filename, index=False)
 for i_c, df in enumerate(pd.read_csv(filename, chunksize=chunks, iterator=True)) :
print(df.shape)
 preds_df = predict_chunk(df_=df,
clfs_=clfs,
features=full_train.columns,
train_mean=train_mean)
 if i_c == 0:
preds_df.to_csv('predictions.csv', header=True, mode='a', index=False)
 else:
preds_df.to_csv('predictions.csv', header=False, mode='a', index=False)
 del preds_df
gc.collect()
print('%15d done in %5.1f minutes' %(chunks *(i_c + 1),(time.time() - start)/ 60), flush=True)
 return
X = training_data.drop(['target', 'seg_id'], axis=1)
 X_test = test_data.drop(['target', 'seg_id'], axis=1)
 test_segs = test_data.seg_id
y = training_data.target<define_variables>
best_params = {
'device': 'cpu',
'objective': 'multiclass',
'num_class': 14,
'boosting_type': 'gbdt',
'n_jobs': -1,
'max_depth': 6,
'n_estimators': 1000,
'subsample_freq': 2,
'subsample_for_bin': 5000,
'min_data_per_group': 100,
'max_cat_to_onehot': 4,
'cat_l2': 1.0,
'cat_smooth': 59.5,
'max_cat_threshold': 32,
'metric_freq': 10,
'verbosity': -1,
'metric': 'multi_logloss',
'xgboost_dart_mode': False,
'uniform_drop': False,
'colsample_bytree': 0.5,
'drop_rate': 0.173,
'learning_rate': 0.025,
'max_drop': 5,
'min_child_samples': 10,
'min_child_weight': 200.0,
'min_split_gain': 0.01,
'num_leaves': 7,
'reg_alpha': 0.1,
'reg_lambda': 0.00023,
'skip_drop': 0.44,
'subsample': 0.75}
folds = KFold(n_splits=5, shuffle=True, random_state=42)
 oof_preds = np.zeros(( len(X), 1))
test_preds = np.zeros(( len(X_test), 1))<init_hyperparams>
trainingDartDf=pd.read_csv('.. /input/writefeaturetablefromsmotedartset/trainingFeatures1039.csv')
 trainingJimsDf=pd.read_csv('.. /input/normalizesomethingdifferentfeatures/traindfNormal.csv')
 if 'Unnamed: 0' in trainingDartDf.columns:
trainingDartDf=trainingDartDf.drop('Unnamed: 0', axis=1)
 print(trainingDartDf.shape)
 columnsToAdd=['outlierScore', 'hipd', 'lipd', 'highEnergy_transitory_1.0_TF',
'highEnergy_transitory_1.5_TF', 'lowEnergy_transitory_1.0_TF',
'lowEnergy_transitory_1.5_TF']
for column in columnsToAdd:
trainingDartDf.loc[:,column]=trainingJimsDf.loc[:,column]
traindf=trainingDartDf
del traindf['hostgal_specz']
del traindf['ra'], traindf['decl'], traindf['gal_l'], traindf['gal_b']
del traindf['ddf']
print(traindf.shape)
 traindf.head()
params = {
"learning_rate": 0.01,
"max_depth": 3,
"n_estimators": 10000,
"min_child_weight": 4,
"colsample_bytree": 1,
"subsample": 0.9,
"nthread": 12,
"random_state": 42
}<find_best_model_class>
testDartDf=pd.read_csv('.. /input/writefeaturetablefromsmotedartset/feat_0.648970_2018-11-23-09-00.csv')
 testJimsDf=pd.read_csv('.. /input/normalizesomethingdifferentfeatures/testdfNormal.csv')
 if 'Unnamed: 0' in testDartDf.columns:
testDartDf=testDartDf.drop('Unnamed: 0', axis=1)
 print(testDartDf.shape)
 testDartDf.head()
for column in columnsToAdd:
testDartDf.loc[:,column]=testJimsDf.loc[:,column]
testdf=testDartDf
del testdf['hostgal_specz']
del testdf['ra'], testdf['decl'], testdf['gal_l'], testdf['gal_b']
del testdf['ddf']
testdf.shape
for fold_,(trn_, val_)in enumerate(folds.split(X)) :
print("Current Fold: {}".format(fold_))
trn_x, trn_y = X.iloc[trn_], y.iloc[trn_]
val_x, val_y = X.iloc[val_], y.iloc[val_]
clf = xgb.XGBRegressor(**params)
 clf.fit(
 trn_x, trn_y,
eval_set=[(trn_x, trn_y),(val_x, val_y)],
eval_metric='mae',
verbose=150,
early_stopping_rounds=100 
)
 val_pred = clf.predict(val_x, ntree_limit=clf.best_ntree_limit)
 test_fold_pred = clf.predict(X_test, ntree_limit=clf.best_ntree_limit)
 print("MAE = {}".format(mean_absolute_error(val_y, val_pred)))
 oof_preds[val_, :] = val_pred.reshape(( -1, 1))
test_preds += test_fold_pred.reshape(( -1, 1))
test_preds /= 5
oof_score = mean_absolute_error(y, oof_preds)
 print("Mean MAE = {}".format(oof_score))<save_to_csv>
for cindex in full_train.columns:
full_train.loc[:,cindex]=np.float64(full_train.loc[:,cindex])
 eval_func = partial(lgbm_modeling_cross_validation,
full_train=full_train,
y=y,
classes=classes,
class_weights=class_weights,
nr_fold=12,
random_state=1)
 best_params.update({'n_estimators': 1000})
 clfs, score = eval_func(best_params )
<import_modules><EOS> 
filename = 'subm_{:.6f}_{}.csv'.format(score,
dt.now().strftime('%Y-%m-%d-%H-%M'))
print('save to {}'.format(filename))
process_test(clfs,
testdf,
full_train,
train_mean=train_mean,
filename=filename,
chunks=40615)
 pdf = pd.read_csv('predictions.csv')
 pdf.to_csv(filename, index=False )