Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<feature_engineering>
import numpy as np
import pandas as pd
from scipy import stats
from scipy import signal
import pywt
import numba
from tqdm.auto import tqdm
from joblib import Parallel, delayed
import gc
str(timedelta(seconds=time_diff))<save_to_csv>
@numba.jit(parallel=True)
 def wavelet_coeffs(x, wavelet='db9', level=9):
coeffs = pywt.wavedec(x, wavelet=wavelet, level=level)
 return coeffs
class FeatureGenerator(object):
def __init__(self, dtype, n_jobs=1, chunk_size=None, fs=4e6, wavelet='db9', level=9):
self.chunk_size = chunk_size
self.dtype = dtype
self.filename = None
self.n_jobs = n_jobs
self.fs = fs
self.wavelet = wavelet
self.level = level
self.test_files = []
if self.dtype == 'train':
self.filename = '.. /input/train.csv'
self.total_data = int(629145481 / self.chunk_size)
 else:
submission = pd.read_csv('.. /input/sample_submission.csv')
 for seg_id in submission.seg_id.values:
self.test_files.append(( seg_id, '.. /input/test/' + seg_id + '.csv'))
self.total_data = int(len(submission))
def read_chunks(self):
if self.dtype == 'train':
iter_df = pd.read_csv(self.filename, iterator=True, chunksize=self.chunk_size,
dtype={'acoustic_data': np.float64, 'time_to_failure': np.float64})
 for counter, df in enumerate(iter_df):
if df.time_to_failure.values[0] > df.time_to_failure.values[-1]:
x = df.acoustic_data.values
y = df.time_to_failure.values[-1]
seg_id = 'train_' + str(counter)
 yield seg_id, x, y
else:
for seg_id, f in self.test_files:
df = pd.read_csv(f, dtype={'acoustic_data': np.float64})
 x = df.acoustic_data.values
yield seg_id, x, -999
@numba.jit(parallel=True)
 def features(self, x, y, seg_id, fs=4e6):
feature_dict = dict()
feature_dict['target'] = y
feature_dict['seg_id'] = seg_id
coeffs = wavelet_coeffs(x, wavelet=self.wavelet, level=self.level)
 coeffs_diff = wavelet_coeffs(np.diff(x), wavelet=self.wavelet, level=self.level)
 percentiles_ranges = [99, 98, 97, 96, 95, 94, 93, 92, 91, 90, 80, 75,
50, 25, 20, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1]
signals_list = {
'regular': coeffs,
'diff': coeffs_diff,
}
for signal_name, signal_i in signals_list.items() :
for i, x_i in enumerate(signal_i):
if i == 0:
name = '{}_cA'.format(signal_name)
 else:
name = '{}_cD{}'.format(signal_name, self.level -(i - 1))
feature_dict['rms_{}'.format(name)] = np.sqrt(np.mean(np.sum(x_i ** 2)))
 feature_dict['mean_{}'.format(name)] = np.mean(x_i)
 feature_dict['median_{}'.format(name)] = np.median(x_i)
 feature_dict['var{}'.format(name)] = np.var(x_i)
 feature_dict['skewnes_{}'.format(name)] = stats.skew(x_i)
 feature_dict['kurtosis_{}'.format(name)] = stats.kurtosis(x_i)
 for m in range(2, 5):
feature_dict['moment_{}_{}'.format(m, name)] = np.mean(np.sum(x_i ** m))
for pct in percentiles_ranges:
feature_dict['percentile{}_{}'.format(str(pct), name)] = np.percentile(x_i, pct)
 chunks = 20
step = len(x_i)// chunks
for chunk_no, band in enumerate(range(0, len(x_i), step)) :
feature_dict['energy_chunk{}_{}'.format(chunk_no, name)] = np.sum(x_i[band:band+step] ** 2)
 feature_dict['energy_chunk_rms{}_{}'.format(chunk_no, name)] = np.sqrt(
 np.mean(
 np.sum(
 feature_dict['energy_chunk{}_{}'.format(chunk_no, name)] ** 2)
)
)
 return feature_dict
def generate(self):
feature_list = []
res = Parallel(n_jobs=self.n_jobs )(delayed(self.features )(x, y, s, fs=self.fs)
 for s, x, y in tqdm(self.read_chunks() , total=self.total_data))
for r in res:
feature_list.append(r)
 return pd.DataFrame(feature_list )
train[['sig_id']+target_cols].to_csv('oof7.csv', index=False)
 valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1] )<save_to_csv>
wavelet = 'db4'
level = 9
training_fg = FeatureGenerator(dtype='train', n_jobs=1, chunk_size=150000, wavelet=wavelet, level=level)
 training_data = training_fg.generate()
test_fg = FeatureGenerator(dtype='test', n_jobs=1, chunk_size=None, wavelet=wavelet, level=level)
 test_data = test_fg.generate()
training_data.to_csv("train_features.csv", index=False)
 test_data.to_csv("test_features.csv", index=False )
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission7.csv', index=False )<categorify>
from sklearn.model_selection import KFold, TimeSeriesSplit
from sklearn.metrics import mean_absolute_error, make_scorer
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
class DenoisingAutoEncoder(nn.Module):
def __init__(self, in_out_channels):
super(DenoisingAutoEncoder, self ).__init__()
self.encoder=nn.Sequential(
 nn.Linear(in_out_channels,256),
nn.ReLU(True),
nn.Linear(256,128),
nn.ReLU(True),
nn.Linear(128,64),
nn.ReLU(True)
)
 self.decoder=nn.Sequential(
 nn.Linear(64,128),
nn.ReLU(True),
nn.Linear(128,256),
nn.ReLU(True),
nn.Linear(256,in_out_channels)
)
 def forward(self,x):
x=self.encoder(x)
 x=self.decoder(x)
 return x<load_from_csv>
features_cols = [c for c in training_data.columns if(( 'target' not in c)and('seg_id' not in c)) ]
X = training_data[features_cols].values
y = training_data['target'].values
X_test = test_data[features_cols].values
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 train_features = pd.merge(train_features, train_drug, on='sig_id')
 train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
groups = np.array(train_features.drug_id.values)
 del train_features['drug_id']
labels_train = labels_train[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/t-test-pca-rfe-logistic-regression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors']<normalization>
from skopt import BayesSearchCV
from skopt.space import Real, Categorical, Integer
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
scaler = preprocessing.StandardScaler()
scaler.fit(train_features[GENES+CELLS].append(test_features[GENES+CELLS]))
data_train = pd.DataFrame(scaler.transform(train_features[GENES+CELLS]), columns=GENES+CELLS)
 data_test = pd.DataFrame(scaler.transform(test_features[GENES+CELLS]), columns=GENES+CELLS)
 data_train.shape, data_test.shape<categorify>
xgb = XGBRegressor(random_state=11)
 search_space = {
'n_estimators': Integer(100, 1000),
'learning_rate': Real(1e-6, 3e-1, 'log-uniform'),
'min_child_weight': Integer(4, 10),
'reg_alpha': Real(1e-6, 0.5, 'log-uniform'),
'reg_lambda': Real(1e-6, 1.0, 'log-uniform'),
'colsample_bytree': Real(0.2, 0.8, 'log-uniform'),
}
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 model = DenoisingAutoEncoder(len(GENES))
model.to(device)
 ae_genes_train_dataset = DAEDataset(
 ids=data_train[GENES].values, 
)
 ae_genes_test_dataset = DAEDataset(
 ids=data_test[GENES].values, 
)
 ae_genes_train_loader = torch.utils.data.DataLoader(
 ae_genes_train_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 ae_genes_test_loader = torch.utils.data.DataLoader(
 ae_genes_test_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 del ae_genes_train_dataset, ae_genes_test_dataset
gc.collect()
model.load_state_dict(torch.load('.. /input/pytorch-moa-dae/GENES_Reconstruction_model.pth'))
ae_genes_train_reconstruction = dae_test_fn(ae_genes_train_loader, model, device)
 ae_genes_test_reconstruction = dae_test_fn(ae_genes_test_loader, model, device)
 print(ae_genes_train_reconstruction.shape)
 print(ae_genes_test_reconstruction.shape)
 model = DenoisingAutoEncoder(len(CELLS))
model.to(device)
 ae_cells_train_dataset = DAEDataset(
 ids=data_train[CELLS].values, 
)
 ae_cells_test_dataset = DAEDataset(
 ids=data_test[CELLS].values, 
)
 ae_cells_train_loader = torch.utils.data.DataLoader(
 ae_cells_train_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 ae_cells_test_loader = torch.utils.data.DataLoader(
 ae_cells_test_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 del ae_cells_train_dataset, ae_cells_test_dataset
gc.collect()
model.load_state_dict(torch.load('.. /input/pytorch-moa-dae/CELLS_Reconstruction_model.pth'))
ae_cells_train_reconstruction = dae_test_fn(ae_cells_train_loader, model, device)
 ae_cells_test_reconstruction = dae_test_fn(ae_cells_test_loader, model, device)
 print(ae_cells_train_reconstruction.shape)
 print(ae_cells_test_reconstruction.shape )<create_dataframe>
opt = BayesSearchCV(
 xgb,
search_space,
scoring=make_scorer(mean_absolute_error, greater_is_better=False),
n_iter=32,
cv=5,
n_jobs=-1,
random_state=11,
refit=True, 
)
 opt.fit(X, y )
ae_cells_train = pd.DataFrame(ae_cells_train_reconstruction, columns=[f'ae_C-{i}' for i in range(64)])
 ae_genes_train = pd.DataFrame(ae_genes_train_reconstruction, columns=[f'ae_G-{i}' for i in range(64)])
 ae_cells_test = pd.DataFrame(ae_cells_test_reconstruction, columns=[f'ae_C-{i}' for i in range(64)])
 ae_genes_test = pd.DataFrame(ae_genes_test_reconstruction, columns=[f'ae_G-{i}' for i in range(64)])
 ae_cells_train.shape, ae_cells_test.shape, ae_genes_train.shape, ae_genes_test.shape<categorify>
print('val.score: {:.3f}'.format(opt.best_score_))
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<categorify>
print(opt.best_params_ )
<compute_test_metric><EOS> 
y_pred = opt.predict(X_test)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['time_to_failure'] = y_pred
sub.to_csv('submission.csv', index=False )