Kaggle competition: PLAsTiCC Astronomical Classification
wavelet = 'db4'
level = 9
training_fg = FeatureGenerator(dtype='train', n_jobs=1, chunk_size=150000, wavelet=wavelet, level=level)
 training_data = training_fg.generate()
test_fg = FeatureGenerator(dtype='test', n_jobs=1, chunk_size=None, wavelet=wavelet, level=level)
 test_data = test_fg.generate()
training_data.to_csv("train_features.csv", index=False)
 test_data.to_csv("test_features.csv", index=False )<import_modules>

 np.warnings.filterwarnings('ignore')
from sklearn.model_selection import KFold, TimeSeriesSplit
from sklearn.metrics import mean_absolute_error, make_scorer
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor<prepare_x_and_y>
def multi_weighted_logloss(y_true, y_preds, classes, class_weights):

 y_p = y_preds.reshape(y_true.shape[0], len(classes), order='F')
 y_ohe = pd.get_dummies(y_true%100)
 y_p = np.clip(a=y_p, a_min=1e-15, a_max=1 - 1e-15)
 y_p_log = np.log(y_p)
 y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)
 nb_pos = y_ohe.sum(axis=0 ).values.astype(float)
 class_arr = np.array([class_weights[k] for k in sorted(class_weights.keys())])
 y_w = y_log_ones * class_arr / nb_pos
loss = - np.sum(y_w)/ np.sum(class_arr)
 return loss
features_cols = [c for c in training_data.columns if(( 'target' not in c)and('seg_id' not in c)) ]
X = training_data[features_cols].values
y = training_data['target'].values
X_test = test_data[features_cols].values<import_modules>
def lgbm_multi_weighted_logloss(y_true, y_preds):

 y_true=y_true%100
y_preds=y_preds%100
classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]
class_weights = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}
loss = multi_weighted_logloss(y_true, y_preds, classes, class_weights)
 return 'wloss', loss, False
def xgb_multi_weighted_logloss(y_predicted, y_true, classes, class_weights):
loss = multi_weighted_logloss(y_true.get_label() , y_predicted,
classes, class_weights)
 return 'wloss', loss
from skopt import BayesSearchCV
from skopt.space import Real, Categorical, Integer<define_search_space>
def save_importances(importances_):
mean_gain = importances_[['gain', 'feature']].groupby('feature' ).mean()
importances_['mean_gain'] = importances_['feature'].map(mean_gain['gain'])
 return importances_
xgb = XGBRegressor(random_state=11)
 search_space = {
'n_estimators': Integer(100, 1000),
'learning_rate': Real(1e-6, 3e-1, 'log-uniform'),
'min_child_weight': Integer(4, 10),
'reg_alpha': Real(1e-6, 0.5, 'log-uniform'),
'reg_lambda': Real(1e-6, 1.0, 'log-uniform'),
'colsample_bytree': Real(0.2, 0.8, 'log-uniform'),
}<train_on_grid>
def smoteAdataset(Xig_train, yig_train, Xig_test, yig_test):
sm=SMOTE(random_state=2)
 Xig_train_res, yig_train_res = sm.fit_sample(Xig_train, yig_train.ravel())
 return Xig_train_res, pd.Series(yig_train_res), Xig_test, pd.Series(yig_test )
opt = BayesSearchCV(
 xgb,
search_space,
scoring=make_scorer(mean_absolute_error, greater_is_better=False),
n_iter=32,
cv=5,
n_jobs=-1,
random_state=11,
refit=True, 
)
 opt.fit(X, y )<compute_test_metric>
def lgbm_modeling_cross_validation(params,
full_train,
y,
classes,
class_weights,
nr_fold=12,
random_state=1):
yy=y%100
w = y.value_counts()
weights = {i : np.sum(w)/ w[i] for i in w.index}
clfs = []
importances = pd.DataFrame()
folds = StratifiedKFold(n_splits=nr_fold,
shuffle=True,
random_state=random_state)
 oof_preds = np.zeros(( len(full_train), np.unique(yy ).shape[0]))
for fold_,(trn_, val_)in enumerate(folds.split(y, y)) :
trn_x, trn_y = full_train.iloc[trn_], y.iloc[trn_]
val_x, val_y = full_train.iloc[val_], y.iloc[val_]
trn_xa, trn_y, val_xa, val_y=smoteAdataset(trn_x.values, trn_y.values, val_x.values, val_y.values)
 trn_y=trn_y%100
val_y=val_y%100
trn_x=pd.DataFrame(data=trn_xa, columns=trn_x.columns)
 val_x=pd.DataFrame(data=val_xa, columns=val_x.columns)
 clf = LGBMClassifier(**params)
 clf.fit(
 trn_x, trn_y,
eval_set=[(trn_x, trn_y),(val_x, val_y)],
eval_metric=lgbm_multi_weighted_logloss,
verbose=100,
early_stopping_rounds=100,
sample_weight=trn_y.map(weights)
)
 clfs.append(clf)
 oof_preds[val_, :] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)
 print('no {}-fold loss: {}'.format(fold_ + 1,
multi_weighted_logloss(val_y, oof_preds[val_, :],
classes, class_weights)))
 imp_df = pd.DataFrame({
'feature': full_train.columns,
'gain': clf.feature_importances_,
'fold': [fold_ + 1] * len(full_train.columns),
})
 importances = pd.concat([importances, imp_df], axis=0, sort=False)
 score = multi_weighted_logloss(y_true=y, y_preds=oof_preds,
classes=classes, class_weights=class_weights)
 print('MULTI WEIGHTED LOG LOSS: {:.5f}'.format(score))
df_importances = save_importances(importances_=importances)
 df_importances.to_csv('lgbm_importances.csv', index=False)
 return clfs, score
print('val.score: {:.3f}'.format(opt.best_score_))<find_best_params>
def predict_chunk(df_, clfs_, features, train_mean):
agg_ = df_
full_test = agg_.reset_index()
print(full_test.head())
 full_test = full_test.fillna(0)
 preds_ = None
for clf in clfs_:
if preds_ is None:
preds_ = clf.predict_proba(full_test[features])/ len(clfs_)
 else:
preds_ += clf.predict_proba(full_test[features])/ len(clfs_)
 preds_99 = np.ones(preds_.shape[0])
 for i in range(preds_.shape[1]):
preds_99 *=(1 - preds_[:, i])
 preds_df_ = pd.DataFrame(preds_, columns=['class_' + str(s)for s in clfs_[0].classes_])
 preds_df_['object_id'] = full_test['object_id']
preds_df_['class_99'] = 0.14 * preds_99 / np.mean(preds_99)
 return preds_df_
print(opt.best_params_ )<save_to_csv>
def process_test(clfs,
testdf,
full_train,
train_mean,
filename='submission.csv',
chunks=40615):
start = time.time()
chunks = 40615
testdf.to_csv(filename, index=False)
 for i_c, df in enumerate(pd.read_csv(filename, chunksize=chunks, iterator=True)) :
print(df.shape)
 preds_df = predict_chunk(df_=df,
clfs_=clfs,
features=full_train.columns,
train_mean=train_mean)
 if i_c == 0:
preds_df.to_csv('predictions.csv', header=True, mode='a', index=False)
 else:
preds_df.to_csv('predictions.csv', header=False, mode='a', index=False)
 del preds_df
gc.collect()
print('%15d done in %5.1f minutes' %(chunks *(i_c + 1),(time.time() - start)/ 60), flush=True)
 return
y_pred = opt.predict(X_test)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['time_to_failure'] = y_pred
sub.to_csv('submission.csv', index=False )<import_modules>
def GenUnknown(data):
return(((((( data["mymedian"])+(((data["mymean"])/ 2.0)))/2.0)) +(((((1.0)-(((data["mymax"])*(((data["mymax"])*(data["mymax"])))))))/ 2.0)))/2.0)
warnings.filterwarnings("ignore" )<load_from_csv>
def compareMeans(traindf, testdf, excludeCols=['target', 'object_id'], sigHigh=.2, sigLow=.4, debug=False):
feats=[]
for col in traindf.columns:
if col not in excludeCols:
travg=np.average(traindf.loc[:,col])
 trstd=np.std(traindf.loc[:,col])
 teavg=np.average(testdf.loc[:,col])
 testd=np.std(testdf.loc[:,col])
 if debug:
tt=ss.ttest_ind(traindf.loc[:,col], testdf.loc[:,col], equal_var=False)
 print(col)
 print(tt)
 print([travg, trstd, teavg, testd])
 if(teavg+testd*sigLow)<(travg-trstd*sigLow):
print('for ' + str(col)+ ' test is smaller than train')
 tt=ss.ttest_ind(traindf.loc[:,col], testdf.loc[:,col], equal_var=False)
 print(col)
 print(tt)
 print([travg, trstd, teavg, testd])
 feats.append(col)
 if(teavg-testd*sigHigh)>(travg+trstd*sigHigh):
print('for ' + str(col)+ ' test is larger than train')
 tt=ss.ttest_ind(traindf.loc[:,col], testdf.loc[:,col], equal_var=False)
 print(col)
 print(tt)
 print([travg, trstd, teavg, testd])
 feats.append(col)
 return feats
%%time
train_df = pd.read_csv(os.path.join(PATH,'train.csv'),
dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})
<create_dataframe>
best_params = {
'device': 'cpu',
'objective': 'multiclass',
'num_class': 14,
'boosting_type': 'gbdt',
'n_jobs': -1,
'max_depth': 6,
'n_estimators': 1000,
'subsample_freq': 2,
'subsample_for_bin': 5000,
'min_data_per_group': 100,
'max_cat_to_onehot': 4,
'cat_l2': 1.0,
'cat_smooth': 59.5,
'max_cat_threshold': 32,
'metric_freq': 10,
'verbosity': -1,
'metric': 'multi_logloss',
'xgboost_dart_mode': False,
'uniform_drop': False,
'colsample_bytree': 0.5,
'drop_rate': 0.173,
'learning_rate': 0.025,
'max_drop': 5,
'min_child_samples': 10,
'min_child_weight': 200.0,
'min_split_gain': 0.01,
'num_leaves': 15,
'reg_alpha': 0.1,
'reg_lambda': 0.00023,
'skip_drop': 0.44,
'subsample': 0.75}
train_df_save = train_df.copy<feature_engineering>
def convertTFToInt(df):
for cindex in df.columns:
if('_TF_' in cindex):
df[cindex]=df[cindex].astype(int)
 if('_TF' in cindex):
df[cindex]=df[cindex].astype(int)
 return df
def removeExtremeValues(df, maxSig=5.88):
for cindex in df.columns:
if cindex not in ['object_id', 'target']:
med=np.median(df[cindex])
 sig=np.std(df[cindex])
 minVal=med-maxSig*sig
maxVal=med+maxSig*sig
highFilter=(df.loc[:,cindex]>maxVal)|(df.loc[:,cindex] == np.inf)
 lowFilter=(df.loc[:,cindex]<minVal)|(df.loc[:,cindex] == -np.inf)
 df.loc[lowFilter,cindex]=minVal
df.loc[highFilter,cindex]=maxVal
return df
def ensureNoNanOrInf(df):
df=df.round(5)
 df.replace(np.inf, 9999, inplace=True)
 for cindex in df.columns:
dropit=df[cindex].isna().sum()
finite=np.isfinite(df[cindex] ).sum()
if finite != df.shape[0]:
print(finite)
 if dropit>0:
print(cindex + ' has ' + str(dropit)+ ' nans')
 df[cindex].fillna(0,inplace=True)
 print(df.shape)
 return df
def cleanupDf(df):
df=ensureNoNanOrInf(df)
 df=convertTFToInt(df)
 df=removeExtremeValues(df, maxSig=5.88)
 return df
mean_acoustic = np.mean(train_df.acoustic_data)
 train_df.acoustic_data = train_df.acoustic_data - mean_acoustic
print(mean_acoustic)
 print(round(np.mean(train_df.acoustic_data),2))<train_model>
trainingDartDf=pd.read_csv('.. /input/writefeaturetablefromsmotedartset/trainingFeatures1039.csv')
 trainingJimsDf=pd.read_csv('.. /input/normalizesomethingdifferentfeatures/traindfNormal.csv')
 if 'Unnamed: 0' in trainingDartDf.columns:
trainingDartDf=trainingDartDf.drop('Unnamed: 0', axis=1)
 print(trainingDartDf.shape)
 columnsToAdd=['outlierScore', 'hipd', 'lipd', 'highEnergy_transitory_1.0_TF',
'highEnergy_transitory_1.5_TF', 'lowEnergy_transitory_1.0_TF',
'lowEnergy_transitory_1.5_TF']
for column in columnsToAdd:
trainingDartDf.loc[:,column]=trainingJimsDf.loc[:,column]
traindf=trainingDartDf
del traindf['hostgal_specz']
del traindf['ra'], traindf['decl'], traindf['gal_l'], traindf['gal_b']
del traindf['ddf']
print(traindf.shape)
 traindf.head()
print("Train: rows:{} cols:{}".format(train_df.shape[0], train_df.shape[1]))<define_variables>
testDartDf=pd.read_csv('.. /input/writefeaturetablefromsmotedartset/feat_0.648970_2018-11-23-09-00.csv')
 testJimsDf=pd.read_csv('.. /input/normalizesomethingdifferentfeatures/testdfNormal.csv')
 if 'Unnamed: 0' in testDartDf.columns:
testDartDf=testDartDf.drop('Unnamed: 0', axis=1)
 print(testDartDf.shape)
 testDartDf.head()
for column in columnsToAdd:
testDartDf.loc[:,column]=testJimsDf.loc[:,column]
testdf=testDartDf
del testdf['hostgal_specz']
del testdf['ra'], testdf['decl'], testdf['gal_l'], testdf['gal_b']
del testdf['ddf']
testdf.shape
rows = 150000
segments = int(np.floor(train_df.shape[0] / rows))
print("Number of segments: ", segments )<normalization>
traindf=cleanupDf(traindf )
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def classic_sta_lta(x, length_sta, length_lta):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]
sta /= length_sta
lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]
lta /= length_lta
sta[:length_lta - 1] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta<set_options>
testdf=cleanupDf(testdf )
gc.collect()<prepare_x_and_y>
feats=compareMeans(traindf, testdf, debug=False, sigHigh=.15, sigLow=.3)
 print(feats )
train_X = pd.DataFrame(index=range(segments), dtype=np.float64)
 train_y = pd.DataFrame(index=range(segments), dtype=np.float64, columns=['time_to_failure'])
 need_aggregated_features = False
if need_aggregated_features:
total_mean = train_df['acoustic_data'].mean()
total_std = train_df['acoustic_data'].std()
total_max = train_df['acoustic_data'].max()
total_min = train_df['acoustic_data'].min()
total_sum = train_df['acoustic_data'].sum()
total_abs_sum = np.abs(train_df['acoustic_data'] ).sum()<feature_engineering>
def medianMatcher(traindf, testdf, theClass, feat, debug=False):
mwtr = np.median(traindf.loc[:,feat])
 mwte = np.median(testdf.loc[:,feat])
 classFilter =(traindf.loc[:,'target']%100)==theClass
mctr = np.median(traindf.loc[classFilter,feat])
 mcte = mctr * mwte / mwtr
aboveFilter =(traindf.loc[:,feat]>mcte)& classFilter
belowFilter =(traindf.loc[:,feat]<mcte)& classFilter
if aboveFilter.sum() > belowFilter.sum() :
traindf.loc[belowFilter,'target'] += 100
else:
traindf.loc[aboveFilter,'target'] += 100
if debug:
print('whole train: ' + str(mwtr))
print('whole test: ' + str(mwte))
print('class train: ' + str(mctr))
print('class test projected: ' + str(mcte))
print('above test class median ' + str(aboveFilter.sum()))
print('below test class median ' + str(belowFilter.sum()))
print(traindf.loc[:,'target'].unique())
 return traindf
def create_features(seg_id, seg, X):
xc = pd.Series(seg['acoustic_data'].values)
 zc = np.fft.fft(xc)
 X.loc[seg_id, 'mean'] = xc.mean()
X.loc[seg_id, 'std'] = xc.std()
X.loc[seg_id, 'max'] = xc.max()
realFFT = np.real(zc)
 imagFFT = np.imag(zc)
 X.loc[seg_id, 'Rmean'] = realFFT.mean()
X.loc[seg_id, 'Rstd'] = realFFT.std()
X.loc[seg_id, 'Rmax'] = realFFT.max()
X.loc[seg_id, 'Rmin'] = realFFT.min()
X.loc[seg_id, 'Imean'] = imagFFT.mean()
X.loc[seg_id, 'Istd'] = imagFFT.std()
X.loc[seg_id, 'Imax'] = imagFFT.max()
X.loc[seg_id, 'Imin'] = imagFFT.min()
X.loc[seg_id, 'Rmean_last_5000'] = realFFT[-5000:].mean()
X.loc[seg_id, 'Rstd__last_5000'] = realFFT[-5000:].std()
X.loc[seg_id, 'Rmax_last_5000'] = realFFT[-5000:].max()
X.loc[seg_id, 'Rmin_last_5000'] = realFFT[-5000:].min()
X.loc[seg_id, 'Rmean_last_15000'] = realFFT[-15000:].mean()
X.loc[seg_id, 'Rstd_last_15000'] = realFFT[-15000:].std()
X.loc[seg_id, 'Rmax_last_15000'] = realFFT[-15000:].max()
X.loc[seg_id, 'Rmin_last_15000'] = realFFT[-15000:].min()
X.loc[seg_id, 'mean_change_abs'] = np.mean(np.diff(xc))
X.loc[seg_id, 'mean_change_rate'] = np.mean(np.nonzero(( np.diff(xc)/ xc[:-1])) [0])
 X.loc[seg_id, 'abs_max'] = np.abs(xc ).max()
X.loc[seg_id, 'abs_min'] = np.abs(xc ).min()
X.loc[seg_id, 'std_first_50000'] = xc[:50000].std()
X.loc[seg_id, 'std_last_50000'] = xc[-50000:].std()
X.loc[seg_id, 'std_first_10000'] = xc[:10000].std()
X.loc[seg_id, 'std_last_10000'] = xc[-10000:].std()
X.loc[seg_id, 'avg_first_50000'] = xc[:50000].mean()
X.loc[seg_id, 'avg_last_50000'] = xc[-50000:].mean()
X.loc[seg_id, 'avg_first_10000'] = xc[:10000].mean()
X.loc[seg_id, 'avg_last_10000'] = xc[-10000:].mean()
X.loc[seg_id, 'min_first_50000'] = xc[:50000].min()
X.loc[seg_id, 'min_last_50000'] = xc[-50000:].min()
X.loc[seg_id, 'min_first_10000'] = xc[:10000].min()
X.loc[seg_id, 'min_last_10000'] = xc[-10000:].min()
X.loc[seg_id, 'max_first_50000'] = xc[:50000].max()
X.loc[seg_id, 'max_last_50000'] = xc[-50000:].max()
X.loc[seg_id, 'max_first_10000'] = xc[:10000].max()
X.loc[seg_id, 'max_last_10000'] = xc[-10000:].max()
X.loc[seg_id, 'max_to_min_diff'] = xc.max() - np.abs(xc.min())
 X.loc[seg_id, 'count_big'] = len(xc[np.abs(xc)> 500])
 X.loc[seg_id, 'sum'] = xc.sum()
X.loc[seg_id, 'mean_change_rate_first_50000'] = np.mean(np.nonzero(( np.diff(xc[:50000])/ xc[:50000][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_last_50000'] = np.mean(np.nonzero(( np.diff(xc[-50000:])/ xc[-50000:][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_first_10000'] = np.mean(np.nonzero(( np.diff(xc[:10000])/ xc[:10000][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_last_10000'] = np.mean(np.nonzero(( np.diff(xc[-10000:])/ xc[-10000:][:-1])) [0])
 X.loc[seg_id, 'q95'] = np.quantile(xc, 0.95)
 X.loc[seg_id, 'q99'] = np.quantile(xc, 0.99)
 X.loc[seg_id, 'q05'] = np.quantile(xc, 0.05)
 X.loc[seg_id, 'q01'] = np.quantile(xc, 0.01)
 X.loc[seg_id, 'abs_q95'] = np.quantile(np.abs(xc), 0.95)
 X.loc[seg_id, 'abs_q99'] = np.quantile(np.abs(xc), 0.99)
 X.loc[seg_id, 'abs_q05'] = np.quantile(np.abs(xc), 0.05)
 X.loc[seg_id, 'abs_q01'] = np.quantile(np.abs(xc), 0.01)
 X.loc[seg_id, 'trend'] = add_trend_feature(xc)
 X.loc[seg_id, 'abs_trend'] = add_trend_feature(xc, abs_values=True)
 X.loc[seg_id, 'abs_mean'] = np.abs(xc ).mean()
X.loc[seg_id, 'abs_std'] = np.abs(xc ).std()
X.loc[seg_id, 'mad'] = xc.mad()
X.loc[seg_id, 'Moving_average_700_mean'] = xc.rolling(window=700 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_1500_mean'] = xc.rolling(window=1500 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_3000_mean'] = xc.rolling(window=3000 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_6000_mean'] = xc.rolling(window=6000 ).mean().mean(skipna=True)
 ewma = pd.Series.ewm
X.loc[seg_id, 'exp_Moving_average_300_mean'] =(ewma(xc, span=300 ).mean() ).mean(skipna=True)
 X.loc[seg_id, 'exp_Moving_average_3000_mean'] = ewma(xc, span=3000 ).mean().mean(skipna=True)
 X.loc[seg_id, 'exp_Moving_average_30000_mean'] = ewma(xc, span=6000 ).mean().mean(skipna=True)
 no_of_std = 2
X.loc[seg_id, 'MA_700MA_std_mean'] = xc.rolling(window=700 ).std().mean()
X.loc[seg_id,'MA_700MA_BB_high_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X.loc[seg_id,'MA_700MA_BB_low_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X.loc[seg_id, 'MA_400MA_std_mean'] = xc.rolling(window=400 ).std().mean()
X.loc[seg_id,'MA_400MA_BB_high_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X.loc[seg_id,'MA_400MA_BB_low_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X.loc[seg_id, 'MA_1000MA_std_mean'] = xc.rolling(window=1000 ).std().mean()
X.loc[seg_id, 'iqr'] = np.subtract(*np.percentile(xc, [75, 25]))
X.loc[seg_id, 'q999'] = np.quantile(xc,0.999)
 X.loc[seg_id, 'q001'] = np.quantile(xc,0.001)
 X.loc[seg_id, 'ave10'] = stats.trim_mean(xc, 0.1)
 for windows in [5, 10, 50, 100, 500, 1000, 5000, 10000]:
x_roll_std = xc.rolling(windows ).std().dropna().values
x_roll_mean = xc.rolling(windows ).mean().dropna().values
X.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X.loc[seg_id, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X.loc[seg_id, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
 X.loc[seg_id, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)
 X.loc[seg_id, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X.loc[seg_id, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X.loc[seg_id, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X.loc[seg_id, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X.loc[seg_id, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X.loc[seg_id, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X.loc[seg_id, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
X.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 X.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 X.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 X.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 X.loc[seg_id, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X.loc[seg_id, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X.loc[seg_id, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max() 
<feature_engineering>
def newTestizeTrain(traindf, testdf, feats, classes=[15, 42, 52, 62, 64, 67, 88, 90, 95]):
for theClass in classes:
for theFeat in feats:
traindf=medianMatcher(traindf, testdf, theClass, theFeat, debug=False)
 return traindf
traindf=newTestizeTrain(traindf, testdf, feats)
 print(traindf.shape)
 print(traindf.loc[:,'target'].unique() )
for seg_id in tqdm_notebook(range(segments)) :
seg = train_df.iloc[seg_id*rows:seg_id*rows+rows]
create_features(seg_id, seg, train_X)
 train_y.loc[seg_id, 'time_to_failure'] = seg['time_to_failure'].values[-1]<prepare_output>
def consolidateClasses(traindf, minSize=7):
for i in traindf.loc[:,'target'].unique() :
theCount=(traindf.loc[:,'target']==i ).sum()
if theCount < minSize:
traindf.loc[(traindf['target']==i),'target']=i%100
print('class ' + str(i)+ ' consolidated due to low(' + str(theCount)+ ')size')
 return traindf
print(len(traindf.loc[:,'target'].unique()))
traindf=consolidateClasses(traindf)
 print(len(traindf.loc[:,'target'].unique()))
train_X_save = train_X.copy
train_y_save = train_y.copy
train_y.head(5)
<save_to_csv>
for cindex in full_train.columns:
full_train.loc[:,cindex]=np.float64(full_train.loc[:,cindex])
 eval_func = partial(lgbm_modeling_cross_validation,
full_train=full_train,
y=y,
classes=classes,
class_weights=class_weights,
nr_fold=12,
random_state=1)
 best_params.update({'n_estimators': 2000})
 clfs, score = eval_func(best_params )
train_y_quake = np.nonzero(np.diff(train_y.time_to_failure)> 0)[0] + 1
print(len(train_y_quake))
print(len(train_y))
for idx in train_y_quake:
train_y.drop([idx],inplace=True)
 train_X.drop([idx],inplace = True)
 train_X.to_csv('train_features.csv', index=False)
 train_y.to_csv('train_y.csv', index=False)
<normalization>
filename = 'subm_{:.6f}_{}.csv'.format(score,
dt.now().strftime('%Y-%m-%d-%H-%M'))
print('save to {}'.format(filename))
process_test(clfs,
testdf,
full_train,
train_mean=train_mean,
filename=filename,
chunks=40615)
 pdf = pd.read_csv('predictions.csv' )
scaler = StandardScaler()
scaler.fit(train_X)
 scaled_train_X = pd.DataFrame(scaler.transform(train_X), columns=train_X.columns)
<load_from_csv>
cols = list(pdf)
 cols.insert(0, cols.pop(cols.index('object_id')))
 pdf = pdf.loc[:, cols]
feats = ['class_6', 'class_15', 'class_16', 'class_42', 'class_52', 'class_53',
'class_62', 'class_64', 'class_65', 'class_67', 'class_88', 'class_90',
'class_92', 'class_95']
y = pd.DataFrame()
y['mymean'] = pdf[feats].mean(axis=1)
 y['mymedian'] = pdf[feats].median(axis=1)
 y['mymax'] = pdf[feats].max(axis=1)
 pdf['class_99'] = GenUnknown(y)
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 test_X = pd.DataFrame(columns=train_X.columns, dtype=np.float64, index=submission.index )<load_from_csv>
def modUnknown(opdf, meta, ddfMult=1.0, mwMult=0.1, preserveMed=False):
pdf=copy.deepcopy(opdf)
 mdf=pdf.merge(meta,on='object_id')
 ddfilter=mdf.loc[:,'ddf']==1
mwfilter=mdf.loc[:,'hostgal_photoz']==0
print(ddfilter.sum())
 print(mwfilter.sum())
 mdf.loc[mwfilter,'class_99']=mwMult*mdf.loc[mwfilter,'class_99']
mdf.loc[ddfilter,'class_99']=ddfMult*mdf.loc[ddfilter,'class_99']
pdf.loc[:,'class_99']=mdf.loc[:,'class_99']
return pdf
for seg_id in tqdm_notebook(test_X.index):
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 seg.acoustic_data = seg.acoustic_data - mean_acoustic
create_features(seg_id, seg, test_X )<save_to_csv>
pdf.to_csv(filename, index=False)