Kaggle competition: PLAsTiCC Astronomical Classification
today = str(datetime.date.today())
 submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 submission["time_to_failure"] =(predictions+NN_predictions+Scirpus_prediction.time_to_failure.values)/3
submission.to_csv(f'FINAL_{today}_submission.csv', index=False)
 submission.head()<set_options>
np.warnings.filterwarnings('ignore')
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
<load_from_csv>

 np.warnings.filterwarnings('ignore')
%%time
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32} )<categorify>
def multi_weighted_logloss(y_true, y_preds, classes, class_weights):

 y_p = y_preds.reshape(y_true.shape[0], len(classes), order='F')
 y_ohe = pd.get_dummies(y_true)
 y_p = np.clip(a=y_p, a_min=1e-15, a_max=1 - 1e-15)
 y_p_log = np.log(y_p)
 y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)
 nb_pos = y_ohe.sum(axis=0 ).values.astype(float)
 class_arr = np.array([class_weights[k] for k in sorted(class_weights.keys())])
 y_w = y_log_ones * class_arr / nb_pos
loss = - np.sum(y_w)/ np.sum(class_arr)
 return loss
etq_meta = [
{"start":0, "end":5656574},
{"start":5656574, "end":50085878},
{"start":50085878, "end":104677356},
{"start":104677356, "end":138772453},
{"start":138772453, "end":187641820},
{"start":187641820, "end":218652630},
{"start":218652630, "end":245829585},
{"start":245829585, "end":307838917},
{"start":307838917, "end":338276287},
{"start":338276287, "end":375377848},
{"start":375377848, "end":419368880},
{"start":419368880, "end":461811623},
{"start":461811623, "end":495800225},
{"start":495800225, "end":528777115},
{"start":528777115, "end":585568144},
{"start":585568144, "end":621985673},
{"start":621985673, "end":629145480},
]
df = []
for i in [2, 7, 0, 4, 11, 13, 9, 1, 14, 10]:
df.append(train[etq_meta[i]['start']:etq_meta[i]['start']+150000*(( etq_meta[i]['end'] - etq_meta[i]['start'])//150000)] )<concatenate>
def lgbm_multi_weighted_logloss(y_true, y_preds):

 classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]
class_weights = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}
loss = multi_weighted_logloss(y_true, y_preds, classes, class_weights)
 return 'wloss', loss, False
def xgb_multi_weighted_logloss(y_predicted, y_true, classes, class_weights):
loss = multi_weighted_logloss(y_true.get_label() , y_predicted,
classes, class_weights)
 return 'wloss', loss
train = pd.concat(df)
 del df
gc.collect()<prepare_x_and_y>
def xgb_modeling_cross_validation(params,
full_train,
y,
classes,
class_weights,
nr_fold=7,
random_state=1):
w = y.value_counts()
weights = {i : np.sum(w)/ w[i] for i in w.index}
func_loss = partial(xgb_multi_weighted_logloss,
classes=classes,
class_weights=class_weights)
 clfs = []
importances = pd.DataFrame()
folds = StratifiedKFold(n_splits=nr_fold,
shuffle=True,
random_state=random_state)
 oof_preds = np.zeros(( len(full_train), np.unique(y ).shape[0]))
for fold_,(trn_, val_)in enumerate(folds.split(y, y)) :
trn_x, trn_y = full_train.iloc[trn_], y.iloc[trn_]
val_x, val_y = full_train.iloc[val_], y.iloc[val_]
clf = XGBClassifier(**params)
 clf.fit(
 trn_x, trn_y,
eval_set=[(trn_x, trn_y),(val_x, val_y)],
eval_metric=func_loss,
verbose=100,
early_stopping_rounds=77,
sample_weight=trn_y.map(weights)
)
 clfs.append(clf)
 oof_preds[val_, :] = clf.predict_proba(val_x, ntree_limit=clf.best_ntree_limit)
 print('no {}-fold loss: {}'.format(fold_ + 1,
multi_weighted_logloss(val_y, oof_preds[val_, :],
classes, class_weights)))
 imp_df = pd.DataFrame({
'feature': full_train.columns,
'gain': clf.feature_importances_,
'fold': [fold_ + 1] * len(full_train.columns),
})
 importances = pd.concat([importances, imp_df], axis=0, sort=False)
 score = multi_weighted_logloss(y_true=y, y_preds=oof_preds,
classes=classes, class_weights=class_weights)
 print('MULTI WEIGHTED LOG LOSS: {:.5f}'.format(score))
df_importances = save_importances(importances_=importances)
 df_importances.to_csv('xgb_importances.csv', index=False)
 return clfs, score
num_seg = len(train)//150000
train_X = []
train_y = []
for i in tqdm_notebook(range(num_seg)) :
if 100000 * i + 150000 < len(train):
train_X.append(train['acoustic_data'].iloc[150000 * i:150000 * i + 150000])
 train_y.append(train['time_to_failure'].iloc[150000 * i + 149999])
 del train
gc.collect()
train_X = np.array(train_X,dtype = np.float32)
 train_y = np.array(train_y,dtype = np.float32 )<feature_engineering>
def smoteAdataset(Xig_train, yig_train, Xig_test, yig_test):
sm=SMOTE(random_state=2)
 Xig_train_res, yig_train_res = sm.fit_sample(Xig_train, yig_train.ravel())
 return Xig_train_res, pd.Series(yig_train_res), Xig_test, pd.Series(yig_test )
X_mean = train_X.mean(0)
 X_std = train_X.std(0)
 train_X -= X_mean
train_X /= X_std
y_mean = train_y.mean()
y_std = train_y.std()
train_y -= y_mean
train_y /= y_std<prepare_x_and_y>
def lgbm_modeling_cross_validation(params,
full_train,
y,
classes,
class_weights,
nr_fold=7,
random_state=1):
w = y.value_counts()
weights = {i : np.sum(w)/ w[i] for i in w.index}
clfs = []
importances = pd.DataFrame()
folds = StratifiedKFold(n_splits=nr_fold,
shuffle=True,
random_state=random_state)
 oof_preds = np.zeros(( len(full_train), np.unique(y ).shape[0]))
for fold_,(trn_, val_)in enumerate(folds.split(y, y)) :
trn_x, trn_y = full_train.iloc[trn_], y.iloc[trn_]
val_x, val_y = full_train.iloc[val_], y.iloc[val_]
trn_xa, trn_y, val_xa, val_y=smoteAdataset(trn_x.values, trn_y.values, val_x.values, val_y.values)
 trn_x=pd.DataFrame(data=trn_xa, columns=trn_x.columns)
 val_x=pd.DataFrame(data=val_xa, columns=val_x.columns)
 clf = LGBMClassifier(**params)
 clf.fit(
 trn_x, trn_y,
eval_set=[(trn_x, trn_y),(val_x, val_y)],
eval_metric=lgbm_multi_weighted_logloss,
verbose=100,
early_stopping_rounds=77,
sample_weight=trn_y.map(weights)
)
 clfs.append(clf)
 oof_preds[val_, :] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)
 print('no {}-fold loss: {}'.format(fold_ + 1,
multi_weighted_logloss(val_y, oof_preds[val_, :],
classes, class_weights)))
 imp_df = pd.DataFrame({
'feature': full_train.columns,
'gain': clf.feature_importances_,
'fold': [fold_ + 1] * len(full_train.columns),
})
 importances = pd.concat([importances, imp_df], axis=0, sort=False)
 score = multi_weighted_logloss(y_true=y, y_preds=oof_preds,
classes=classes, class_weights=class_weights)
 print('MULTI WEIGHTED LOG LOSS: {:.5f}'.format(score))
df_importances = save_importances(importances_=importances)
 df_importances.to_csv('lgbm_importances.csv', index=False)
 return clfs, score, oof_preds, importances
<load_from_csv>
def predict_chunk(df_, clfs_, features, train_mean):
agg_ = df_
full_test = agg_.reset_index()
print(full_test.head())
 full_test = full_test.fillna(0)
 preds_ = None
for clf in clfs_:
if preds_ is None:
preds_ = clf.predict_proba(full_test[features])/ len(clfs_)
 else:
preds_ += clf.predict_proba(full_test[features])/ len(clfs_)
 preds_99 = np.ones(preds_.shape[0])
 for i in range(preds_.shape[1]):
preds_99 *=(1 - preds_[:, i])
 preds_df_ = pd.DataFrame(preds_, columns=['class_' + str(s)for s in clfs_[0].classes_])
 preds_df_['object_id'] = full_test['object_id']
preds_df_['class_99'] = 0.14 * preds_99 / np.mean(preds_99)
 return preds_df_
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 test_X = []
for i, seg_id in enumerate(tqdm_notebook(submission.index)) :
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 test_X.append(seg['acoustic_data'])
 test_X = np.array(test_X,dtype = np.float32)
 test_X -= X_mean
test_X /= X_std<concatenate>
def process_test(clfs,
testdf,
full_train,
train_mean,
filename='submission.csv',
chunks=40615):
start = time.time()
chunks = 40615
testdf=testdf.round(4)
 testdf.to_csv(filename, index=False)
 for i_c, df in enumerate(pd.read_csv(filename, chunksize=chunks, iterator=True)) :
print(df.shape)
 preds_df = predict_chunk(df_=df,
clfs_=clfs,
features=full_train.columns,
train_mean=train_mean)
 if i_c == 0:
preds_df.to_csv('predictions.csv', header=True, mode='a', index=False)
 else:
preds_df.to_csv('predictions.csv', header=False, mode='a', index=False)
 del preds_df
gc.collect()
print('%15d done in %5.1f minutes' %(chunks *(i_c + 1),(time.time() - start)/ 60), flush=True)
 return
train_X = np.expand_dims(train_X,-1)
 test_X = np.expand_dims(test_X,-1 )<set_options>
def GenUnknown(data):
return(((((( data["mymedian"])+(((data["mymean"])/ 2.0)))/2.0)) +(((((1.0)-(((data["mymax"])*(((data["mymax"])*(data["mymax"])))))))/ 2.0)))/2.0)
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<categorify>
best_params = {
'device': 'cpu',
'objective': 'multiclass',
'num_class': 14,
'boosting_type': 'gbdt',
'n_jobs': -1,
'max_depth': 7,
'n_estimators': 1000,
'subsample_freq': 2,
'subsample_for_bin': 5000,
'min_data_per_group': 200,
'max_cat_to_onehot': 4,
'cat_l2': 1.0,
'cat_smooth': 59.5,
'max_cat_threshold': 32,
'metric_freq': 10,
'verbosity': -1,
'metric': 'multi_logloss',
'xgboost_dart_mode': False,
'uniform_drop': False,
'colsample_bytree': 0.5,
'drop_rate': 0.173,
'learning_rate': 0.0133,
'max_drop': 5,
'min_child_samples': 20,
'min_child_weight': 100.0,
'min_split_gain': 0.1,
'num_leaves': 7,
'reg_alpha': 0.1,
'reg_lambda': 0.00023,
'skip_drop': 0.44,
'subsample': 0.75}
def keras_model() :
def wave_block(x,filters,kernel_size,n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters=filters,
kernel_size=1,
padding='same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = Conv1D(filters=filters,
kernel_size=kernel_size,
padding='same',
activation = 'tanh',
dilation_rate=dilation_rate )(x)
 sigm_out = Conv1D(filters=filters,
kernel_size=kernel_size,
padding='same',
activation = 'sigmoid',
dilation_rate=dilation_rate )(x)
 x = Multiply()([tanh_out,sigm_out])
 x = Conv1D(filters = filters,
kernel_size = 1,
padding='same' )(x)
 res_x = Add()([res_x,x])
 return res_x
clear_session()
inp = Input(shape=(150000,1))
Collect = []
x = wave_block(inp,16,3,8)
 x = AveragePooling1D(10 )(x)
 x = wave_block(x,32,3,5)
 x = AveragePooling1D(10 )(x)
 x = wave_block(x,64,3,3)
 x = AveragePooling1D(10 )(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(150 )(x)
 x = Dropout(0.2 )(x)
 x = Dense(128, activation="relu" )(x)
 x = Dense(1 )(x)
 model = Model(inputs = inp, outputs=x)
 return model
model = keras_model()
model.summary() 
<normalization>
trainingDartDf=pd.read_csv('.. /input/writefeaturetablefromsmotedartset/trainingFeatures1039.csv')
 trainingJimsDf=pd.read_csv('.. /input/aggregatecustomfeatures/trainingAggregate.csv')
 if 'Unnamed: 0' in trainingDartDf.columns:
trainingDartDf=trainingDartDf.drop('Unnamed: 0', axis=1)
 print(trainingDartDf.shape)
def step_decay(epoch):
x = 0.0001
if epoch >= 40: x = 0.00001
return x
lr_decay = LearningRateScheduler(step_decay )<save_to_csv>
columnsToAdd=['pb0maxstd', 'pb0medmed', 'pb0maxavg', 'pb0maxdsl',
'pb3maxstd', 'pb3medmed', 'pb3maxavg', 'pb3maxdsl', 'pb3transitory',
'pb0transitory', 'pb1maxstd', 'pb1medmed', 'pb1maxavg', 'pb1maxdsl',
'pb4maxstd', 'pb4medmed', 'pb4maxavg', 'pb4maxdsl', 'pb4transitory',
'pb1transitory', 'pb2maxstd', 'pb2medmed', 'pb2maxavg', 'pb2maxdsl',
'pb5maxstd', 'pb5medmed', 'pb5maxavg', 'pb5maxdsl', 'pb5transitory',
'pb2transitory', 'irmaxstd', 'vismaxstd', 'uvmaxstd', 'irmedmed',
'vismedmed', 'uvmedmed', 'irmaxavg', 'vismaxavg', 'uvmaxavg',
'irmaxdsl', 'vismaxdsl', 'uvmaxdsl', 'irtransitory', 'vistransitory',
'uvtransitory', 'irSpread', 'uvSpread', 'visSpread', 'irUvSpreadRatio',
'irMinusUvTransitory', 'irVisSpreadRatio', 'irMinusVisTransitory']
for column in columnsToAdd:
trainingDartDf.loc[:,column]=trainingJimsDf.loc[:,column]
traindf=trainingDartDf
del traindf['hostgal_specz']
del traindf['ra'], traindf['decl'], traindf['gal_l'], traindf['gal_b']
del traindf['ddf']
print(traindf.shape)
 traindf.head()
prediction = np.zeros(len(test_X))
model = keras_model()
model.compile(Adam(lr=0.00001), loss='mse', metrics=['mae'])
 model.fit(train_X,train_y,
batch_size = 16,
epochs=100,
verbose=2 
)
 prediction = model.predict(test_X)[:,0]
submission['time_to_failure'] = prediction * y_std + y_mean
print(submission.head())
 submission.to_csv('submission.csv' )<prepare_x_and_y>
testDartDf=pd.read_csv('.. /input/writefeaturetablefromsmotedartset/feat_0.648970_2018-11-23-09-00.csv')
 testJimsDf=pd.read_csv('.. /input/fork-of-aggregatecustomfeaturestest/testAggregate.csv')
 if 'Unnamed: 0' in testDartDf.columns:
testDartDf=testDartDf.drop('Unnamed: 0', axis=1)
 print(testDartDf.shape)
 testDartDf.head()
for column in columnsToAdd:
testDartDf[column]=testJimsDf[column]
testdf=testDartDf
del testdf['hostgal_specz']
del testdf['ra'], testdf['decl'], testdf['gal_l'], testdf['gal_b']
del testdf['ddf']
testdf.shape
<set_options>
for cindex in full_train.columns:
full_train.loc[:,cindex]=np.float64(full_train.loc[:,cindex])
 eval_func = partial(lgbm_modeling_cross_validation,
full_train=full_train,
y=y,
classes=classes,
class_weights=class_weights,
nr_fold=5,
random_state=1)
 best_params.update({'n_estimators': 2000})
 clfs, score, oof_preds, importances = eval_func(best_params )
seed(639)
 set_random_seed(5944)
 seed(639)
 set_random_seed(5944)
 float_data = pd.read_csv(".. /input/train.csv", dtype={"acoustic_data": np.float32, "time_to_failure": np.float32} ).values
def extract_features(z):
return np.c_[z.mean(axis=1),
z.min(axis=1),
z.max(axis=1),
z.std(axis=1)]
def create_X(x, last_index=None, n_steps=150, step_length=1000):
if last_index == None:
last_index=len(x)
 assert last_index - n_steps * step_length >= 0
temp =(x[(last_index - n_steps * step_length):last_index].reshape(n_steps, -1)- 5)/ 3
return np.c_[extract_features(temp),
extract_features(temp[:, -step_length // 10:]),
extract_features(temp[:, -step_length // 100:]),
extract_features(temp[:, -step_length // 300:])]
n_features = create_X(float_data[0:150000] ).shape[1]
print("Our RNN is based on %i features"% n_features)
 def generator(data, min_index=0, max_index=None, batch_size=16, n_steps=150, step_length=1000):
if max_index is None:
max_index = len(data)- 1
while True:
rows = np.random.randint(min_index + n_steps * step_length, max_index, size=batch_size)
 samples = np.zeros(( batch_size, n_steps, n_features))
targets = np.zeros(batch_size,)
 for j, row in enumerate(rows):
samples[j] = create_X(data[:, 0], last_index=row, n_steps=n_steps, step_length=step_length)
 targets[j] = data[row - 1, 1]
yield samples, targets
batch_size = 32
second_earthquake = 50085877
float_data[second_earthquake, 1]
train_gen = generator(float_data, batch_size=batch_size)
 valid_gen = generator(float_data, batch_size=batch_size, max_index=second_earthquake )<choose_model_class>
filename = 'subm_{:.6f}_{}.csv'.format(score,
dt.now().strftime('%Y-%m-%d-%H-%M'))
print('save to {}'.format(filename))
process_test(clfs,
testdf,
full_train,
train_mean=train_mean,
filename=filename,
chunks=40615)
 pdf = pd.read_csv('predictions.csv' )
<load_from_csv><EOS> 
cols = list(pdf)
 cols.insert(0, cols.pop(cols.index('object_id')))
 pdf = pdf.loc[:, cols]
feats = ['class_6', 'class_15', 'class_16', 'class_42', 'class_52', 'class_53',
'class_62', 'class_64', 'class_65', 'class_67', 'class_88', 'class_90',
'class_92', 'class_95']
y = pd.DataFrame()
y['mymean'] = pdf[feats].mean(axis=1)
 y['mymedian'] = pdf[feats].median(axis=1)
 y['mymax'] = pdf[feats].max(axis=1)
 pdf['class_99'] = GenUnknown(y)
 pdf.to_csv(filename, index=False)