Kaggle competition: LANL Earthquake Prediction
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
train_features = rank_gauss(train_features)
 test_features = rank_gauss(test_features)
 data_train = train_features[GENES+CELLS]
data_test = test_features[GENES+CELLS]<categorify>
%%time
train_df = pd.read_csv(os.path.join(PATH,'train.csv'),nrows = 6000000,dtype={
'acoustic_data': np.int16, 'time_to_failure': np.float32} )
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 model = DenoisingAutoEncoder(len(GENES), 512)
 model.to(device)
 ae_genes_train_dataset = DAEDataset(
 ids=data_train[GENES].values, 
)
 ae_genes_test_dataset = DAEDataset(
 ids=data_test[GENES].values, 
)
 ae_genes_train_loader = torch.utils.data.DataLoader(
 ae_genes_train_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 ae_genes_test_loader = torch.utils.data.DataLoader(
 ae_genes_test_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 del ae_genes_train_dataset, ae_genes_test_dataset
gc.collect()
model.load_state_dict(torch.load('.. /input/pytorch-moa-dae-v5/GENES_Reconstruction_model.pth'))
ae_genes_train_reconstruction = dae_test_fn(ae_genes_train_loader, model, device)
 ae_genes_test_reconstruction = dae_test_fn(ae_genes_test_loader, model, device)
 print(ae_genes_train_reconstruction.shape)
 print(ae_genes_test_reconstruction.shape )<categorify>
x = train_df['acoustic_data'].values[::100]
y = train_df['time_to_failure'].values[::100]
model = DenoisingAutoEncoder(len(CELLS), 64)
 model.to(device)
 ae_cells_train_dataset = DAEDataset(
 ids=data_train[CELLS].values, 
)
 ae_cells_test_dataset = DAEDataset(
 ids=data_test[CELLS].values, 
)
 ae_cells_train_loader = torch.utils.data.DataLoader(
 ae_cells_train_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 ae_cells_test_loader = torch.utils.data.DataLoader(
 ae_cells_test_dataset,
shuffle=False,
batch_size=128,
num_workers=0,
pin_memory=True 
)
 del ae_cells_train_dataset, ae_cells_test_dataset
gc.collect()
model.load_state_dict(torch.load('.. /input/pytorch-moa-dae-v5/CELLS_Reconstruction_model.pth'))
ae_cells_train_reconstruction = dae_test_fn(ae_cells_train_loader, model, device)
 ae_cells_test_reconstruction = dae_test_fn(ae_cells_test_loader, model, device)
 print(ae_cells_train_reconstruction.shape)
 print(ae_cells_test_reconstruction.shape )<create_dataframe>
train = pd.read_csv(os.path.join(PATH,'train.csv'), iterator=True, chunksize=150_000, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})
 X_train = pd.DataFrame()
y_train = pd.Series()
for df in train:
ch = gen_features(df['acoustic_data'])
 X_train = X_train.append(ch, ignore_index=True)
 y_train = y_train.append(pd.Series(df['time_to_failure'].values[-1]))
ae_cells_train = pd.DataFrame(ae_cells_train_reconstruction, columns=[f'ae_C-{i}' for i in range(64//4)])
 ae_genes_train = pd.DataFrame(ae_genes_train_reconstruction, columns=[f'ae_G-{i}' for i in range(512//4)])
 ae_cells_test = pd.DataFrame(ae_cells_test_reconstruction, columns=[f'ae_C-{i}' for i in range(64//4)])
 ae_genes_test = pd.DataFrame(ae_genes_test_reconstruction, columns=[f'ae_G-{i}' for i in range(512//4)])
 ae_cells_train.shape, ae_cells_test.shape, ae_genes_train.shape, ae_genes_test.shape<create_dataframe>
from catboost import CatBoostRegressor, Pool
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV
from sklearn.svm import NuSVR, SVR
from sklearn.kernel_ridge import KernelRidge
n_comp_GENES = 450
n_comp_CELLS = 2
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(TruncatedSVD(n_components=n_comp_GENES, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'svd_G-{i}' for i in range(n_comp_GENES)])
 test2 = pd.DataFrame(test2, columns=[f'svd_G-{i}' for i in range(n_comp_GENES)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(TruncatedSVD(n_components=n_comp_CELLS, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'svd_C-{i}' for i in range(n_comp_CELLS)])
 test2 = pd.DataFrame(test2, columns=[f'svd_C-{i}' for i in range(n_comp_CELLS)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<concatenate>
train_pool = Pool(X_train,y_train)
 m= CatBoostRegressor(iterations=10000,loss_function='MAE',boosting_type='Ordered')
 m.fit(X_train,y_train,silent = True)
 m.best_score_
train_features = pd.concat(( train_features, ae_genes_train, ae_cells_train), axis=1)
 test_features = pd.concat(( test_features, ae_genes_test, ae_cells_test), axis=1)
 train_features.shape, test_features.shape<create_dataframe>
scaler = StandardScaler()
scaler.fit(X_train)
 X_train_scaled = scaler.transform(X_train)
var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape<merge>
m2 = SVR(kernel = 'rbf', tol = 0.01, C= 2, gamma=0.02)
 m2.fit(X_train_scaled, y_train.values.flatten() )
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 train = train[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
 target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 print('num_targets: {}'.format(num_targets))
print('num_aux_targets: {}'.format(num_aux_targets))
print('num_all_targets: {}'.format(num_all_targets))<choose_model_class>
PATH1 = os.path.join(PATH,'test' )
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x<choose_model_class>
X_test = pd.DataFrame()
y_test = pd.Series()
for file in os.listdir(os.path.join(PATH,'test')) :
data = pd.read_csv(os.path.join(PATH1,file), dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})
 data_fet = gen_features(data['acoustic_data'])
 X_test = X_test.append(data_fet,ignore_index = True)
 y_test = y_test.append(pd.Series(os.path.splitext(file)[0]))
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]<define_variables>
X_test_scaled = scaler.transform(X_test)
 y_pred_svr = m2.predict(X_test_scaled )
feature_cols = [c for c in process_data(train ).columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 num_features<define_variables>
sub = pd.DataFrame(columns=['seg_id','time_to_failure'])
 sub['seg_id'] = y_test
sub['time_to_failure'] = y_pred_svr
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1<choose_model_class>
filename = 'submission1.csv'
sub.to_csv(filename,index=False)
 print('Saved file: ' + filename )