Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<choose_model_class>
warnings.filterwarnings("ignore" )
def build_model(n_features, n_features_2, n_labels, label_smoothing = 0.0005):
input_1 = layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.2),
layers.Dense(512, activation="elu"),
layers.BatchNormalization() ,
layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = layers.Concatenate()([input_2, input_3])
 head_2 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.3),
layers.Dense(512, "relu"),
layers.BatchNormalization() ,
layers.Dense(512, "elu"),
layers.BatchNormalization() ,
layers.Dense(256, "relu"),
layers.BatchNormalization() ,
layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = layers.Average()([input_3, input_4])
 head_3 = Sequential([
layers.BatchNormalization() ,
layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=logloss)
 return model<prepare_x_and_y>
%%time
train_df = pd.read_csv(os.path.join(PATH,'train.csv'),
dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})
n_seeds = 3
np.random.seed(1)
 seeds = np.random.randint(0,100,size=n_seeds)
 n_folds = 7
y_pred = np.zeros(( n_test,n_labels))
oof = tf.constant(0.0)
 oof_pred = np.zeros(( n_train,n_labels))
hists = []
for seed in seeds:
fold = 0
kf = MultilabelStratifiedKFold(n_splits=n_folds)
 for train, test in kf.split(X=train_features, y=labels_train, groups=groups):
X_train, X_test = preprocessor(train_features.iloc[train].values,
train_features.iloc[test].values)
 _,data_test = preprocessor(train_features.iloc[train].values,
test_features.drop('cp_type',axis=1 ).values)
 X_train_2 = train_features.iloc[train][predictors].values
X_test_2 = train_features.iloc[test][predictors].values
data_test_2 = test_features[predictors].values
X_train_2 = np.concatenate([X_train_2,
ae_genes_train.iloc[train, :].values,
ae_cells_train.iloc[train, :].values], 1)
 X_test_2 = np.concatenate([X_test_2,
ae_genes_train.iloc[test, :].values,
ae_cells_train.iloc[test, :].values], 1)
 data_test_2 = np.concatenate([data_test_2,
ae_genes_test.values,
ae_cells_test.values], 1)
 y_train = labels_train[train]
y_test = labels_train[test]
n_features = X_train.shape[1]
n_features_2 = X_train_2.shape[1]
model = build_model(n_features, n_features_2, n_labels)
 reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_logloss', factor=0.1, patience=2, mode='min', min_lr=1E-5)
 early_stopping = callbacks.EarlyStopping(monitor='val_logloss', min_delta=1E-5, patience=10, mode='min',restore_best_weights=True)
 hist = model.fit([X_train,X_train_2],y_train, batch_size=128, epochs=192,verbose=1,validation_data =([X_test,X_test_2],y_test),
callbacks=[reduce_lr, early_stopping])
 hists.append(hist)
 model.save('TwoHeads_seed_'+str(seed)+'_fold_'+str(fold))
y_val = model.predict([X_test,X_test_2])
 oof += logloss(tf.constant(y_test,dtype=tf.float32),tf.constant(y_val,dtype=tf.float32)) /(n_folds*n_seeds)
 oof_pred[test] += y_val/(n_folds*n_seeds)
 y_pred += model.predict([data_test,data_test_2])/(n_folds*n_seeds)
 fold += 1<save_to_csv>
train_df_save = train_df.copy
oof_train = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv' ).loc[non_ctl_idx, :].reset_index(drop=True)
 oof_train.iloc[:, 1:] = oof_pred
oof_train.to_csv('oof8.csv', index=False )<save_to_csv>
mean_acoustic = np.mean(train_df.acoustic_data)
 train_df.acoustic_data = train_df.acoustic_data - mean_acoustic
print(mean_acoustic)
 print(round(np.mean(train_df.acoustic_data),2))
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] = np.clip(y_pred,p_min,p_max)
 sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub.to_csv('submission8.csv', index=False )<load_pretrained>
print("Train: rows:{} cols:{}".format(train_df.shape[0], train_df.shape[1]))
<load_from_csv>
rows = 150000
segments = int(np.floor(train_df.shape[0] / rows))
print("Number of segments: ", segments )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 df = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train_features2=train_features.copy()
test_features2=test_features.copy()
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<categorify>
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def classic_sta_lta(x, length_sta, length_lta):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]
sta /= length_sta
lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]
lta /= length_lta
sta[:length_lta - 1] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<create_dataframe>
gc.collect()
n_comp = 600
pca_g = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 gpca=(pca_g.fit(data[GENES]))
train2=(gpca.transform(train_features[GENES]))
test2 =(gpca.transform(test_features[GENES]))
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1)
 n_comp = 50
pca_c = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 cpca=(pca_c.fit(data[CELLS]))
train2=(cpca.transform(train_features[CELLS]))
test2 =(cpca.transform(test_features[CELLS]))
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1 )<concatenate>
train_X = pd.DataFrame(index=range(segments), dtype=np.float64)
 train_y = pd.DataFrame(index=range(segments), dtype=np.float64, columns=['time_to_failure'])
 need_aggregated_features = False
if need_aggregated_features:
total_mean = train_df['acoustic_data'].mean()
total_std = train_df['acoustic_data'].std()
total_max = train_df['acoustic_data'].max()
total_min = train_df['acoustic_data'].min()
total_sum = train_df['acoustic_data'].sum()
total_abs_sum = np.abs(train_df['acoustic_data'] ).sum()
c_n = [f for f in list(train_features.columns)if f not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train_features[c_n].var() >= 0.85 ).values
tmp = train_features[c_n].loc[:, mask]
train_features = pd.concat([train_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 tmp = test_features[c_n].loc[:, mask]
test_features = pd.concat([test_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1 )<find_best_model_class>
def create_features(seg_id, seg, X):
xc = pd.Series(seg['acoustic_data'].values)
 zc = np.fft.fft(xc)
 X.loc[seg_id, 'mean'] = xc.mean()
X.loc[seg_id, 'std'] = xc.std()
X.loc[seg_id, 'max'] = xc.max()
realFFT = np.real(zc)
 imagFFT = np.imag(zc)
 X.loc[seg_id, 'Rmean'] = realFFT.mean()
X.loc[seg_id, 'Rstd'] = realFFT.std()
X.loc[seg_id, 'Rmax'] = realFFT.max()
X.loc[seg_id, 'Rmin'] = realFFT.min()
X.loc[seg_id, 'Imean'] = imagFFT.mean()
X.loc[seg_id, 'Istd'] = imagFFT.std()
X.loc[seg_id, 'Imax'] = imagFFT.max()
X.loc[seg_id, 'Imin'] = imagFFT.min()
X.loc[seg_id, 'Rmean_last_5000'] = realFFT[-5000:].mean()
X.loc[seg_id, 'Rstd__last_5000'] = realFFT[-5000:].std()
X.loc[seg_id, 'Rmax_last_5000'] = realFFT[-5000:].max()
X.loc[seg_id, 'Rmin_last_5000'] = realFFT[-5000:].min()
X.loc[seg_id, 'Rmean_last_15000'] = realFFT[-15000:].mean()
X.loc[seg_id, 'Rstd_last_15000'] = realFFT[-15000:].std()
X.loc[seg_id, 'Rmax_last_15000'] = realFFT[-15000:].max()
X.loc[seg_id, 'Rmin_last_15000'] = realFFT[-15000:].min()
X.loc[seg_id, 'mean_change_abs'] = np.mean(np.diff(xc))
X.loc[seg_id, 'mean_change_rate'] = np.mean(np.nonzero(( np.diff(xc)/ xc[:-1])) [0])
 X.loc[seg_id, 'abs_max'] = np.abs(xc ).max()
X.loc[seg_id, 'abs_min'] = np.abs(xc ).min()
X.loc[seg_id, 'std_first_50000'] = xc[:50000].std()
X.loc[seg_id, 'std_last_50000'] = xc[-50000:].std()
X.loc[seg_id, 'std_first_10000'] = xc[:10000].std()
X.loc[seg_id, 'std_last_10000'] = xc[-10000:].std()
X.loc[seg_id, 'avg_first_50000'] = xc[:50000].mean()
X.loc[seg_id, 'avg_last_50000'] = xc[-50000:].mean()
X.loc[seg_id, 'avg_first_10000'] = xc[:10000].mean()
X.loc[seg_id, 'avg_last_10000'] = xc[-10000:].mean()
X.loc[seg_id, 'min_first_50000'] = xc[:50000].min()
X.loc[seg_id, 'min_last_50000'] = xc[-50000:].min()
X.loc[seg_id, 'min_first_10000'] = xc[:10000].min()
X.loc[seg_id, 'min_last_10000'] = xc[-10000:].min()
X.loc[seg_id, 'max_first_50000'] = xc[:50000].max()
X.loc[seg_id, 'max_last_50000'] = xc[-50000:].max()
X.loc[seg_id, 'max_first_10000'] = xc[:10000].max()
X.loc[seg_id, 'max_last_10000'] = xc[-10000:].max()
X.loc[seg_id, 'max_to_min_diff'] = xc.max() - np.abs(xc.min())
 X.loc[seg_id, 'count_big'] = len(xc[np.abs(xc)> 500])
 X.loc[seg_id, 'sum'] = xc.sum()
X.loc[seg_id, 'mean_change_rate_first_50000'] = np.mean(np.nonzero(( np.diff(xc[:50000])/ xc[:50000][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_last_50000'] = np.mean(np.nonzero(( np.diff(xc[-50000:])/ xc[-50000:][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_first_10000'] = np.mean(np.nonzero(( np.diff(xc[:10000])/ xc[:10000][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_last_10000'] = np.mean(np.nonzero(( np.diff(xc[-10000:])/ xc[-10000:][:-1])) [0])
 X.loc[seg_id, 'q95'] = np.quantile(xc, 0.95)
 X.loc[seg_id, 'q99'] = np.quantile(xc, 0.99)
 X.loc[seg_id, 'q05'] = np.quantile(xc, 0.05)
 X.loc[seg_id, 'q01'] = np.quantile(xc, 0.01)
 X.loc[seg_id, 'abs_q95'] = np.quantile(np.abs(xc), 0.95)
 X.loc[seg_id, 'abs_q99'] = np.quantile(np.abs(xc), 0.99)
 X.loc[seg_id, 'abs_q05'] = np.quantile(np.abs(xc), 0.05)
 X.loc[seg_id, 'abs_q01'] = np.quantile(np.abs(xc), 0.01)
 X.loc[seg_id, 'trend'] = add_trend_feature(xc)
 X.loc[seg_id, 'abs_trend'] = add_trend_feature(xc, abs_values=True)
 X.loc[seg_id, 'abs_mean'] = np.abs(xc ).mean()
X.loc[seg_id, 'abs_std'] = np.abs(xc ).std()
X.loc[seg_id, 'mad'] = xc.mad()
X.loc[seg_id, 'Moving_average_700_mean'] = xc.rolling(window=700 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_1500_mean'] = xc.rolling(window=1500 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_3000_mean'] = xc.rolling(window=3000 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_6000_mean'] = xc.rolling(window=6000 ).mean().mean(skipna=True)
 ewma = pd.Series.ewm
X.loc[seg_id, 'exp_Moving_average_300_mean'] =(ewma(xc, span=300 ).mean() ).mean(skipna=True)
 X.loc[seg_id, 'exp_Moving_average_3000_mean'] = ewma(xc, span=3000 ).mean().mean(skipna=True)
 X.loc[seg_id, 'exp_Moving_average_30000_mean'] = ewma(xc, span=6000 ).mean().mean(skipna=True)
 no_of_std = 2
X.loc[seg_id, 'MA_700MA_std_mean'] = xc.rolling(window=700 ).std().mean()
X.loc[seg_id,'MA_700MA_BB_high_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X.loc[seg_id,'MA_700MA_BB_low_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X.loc[seg_id, 'MA_400MA_std_mean'] = xc.rolling(window=400 ).std().mean()
X.loc[seg_id,'MA_400MA_BB_high_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X.loc[seg_id,'MA_400MA_BB_low_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X.loc[seg_id, 'MA_1000MA_std_mean'] = xc.rolling(window=1000 ).std().mean()
X.loc[seg_id, 'iqr'] = np.subtract(*np.percentile(xc, [75, 25]))
X.loc[seg_id, 'q999'] = np.quantile(xc,0.999)
 X.loc[seg_id, 'q001'] = np.quantile(xc,0.001)
 X.loc[seg_id, 'ave10'] = stats.trim_mean(xc, 0.1)
 for windows in [5, 10, 50, 100, 500, 1000, 5000, 10000]:
x_roll_std = xc.rolling(windows ).std().dropna().values
x_roll_mean = xc.rolling(windows ).mean().dropna().values
X.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X.loc[seg_id, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X.loc[seg_id, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
 X.loc[seg_id, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)
 X.loc[seg_id, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X.loc[seg_id, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X.loc[seg_id, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X.loc[seg_id, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X.loc[seg_id, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X.loc[seg_id, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X.loc[seg_id, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
X.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 X.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 X.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 X.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 X.loc[seg_id, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X.loc[seg_id, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X.loc[seg_id, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max()
def fe_cluster_genes(train, test, n_clusters_g = 22, SEED = 42):
features_g = GENES
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans_genes = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans_genes.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_genes.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 return train, test
train_features2 ,test_features2=fe_cluster_genes(train_features2,test_features2 )<find_best_model_class>
for seg_id in tqdm_notebook(range(segments)) :
seg = train_df.iloc[seg_id*rows:seg_id*rows+rows]
create_features(seg_id, seg, train_X)
 train_y.loc[seg_id, 'time_to_failure'] = seg['time_to_failure'].values[-1]
def fe_cluster_cells(train, test, n_clusters_c = 4, SEED = 42):
features_c = CELLS
def create_cluster(train, test, features, kind = 'c', n_clusters = n_clusters_c):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans_cells = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans_cells.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_cells.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster_cells(train_features2,test_features2 )<concatenate>
train_X_save = train_X.copy
train_y_save = train_y.copy
train_y.head(5)
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )<predict_on_test>
train_y_quake = np.nonzero(np.diff(train_y.time_to_failure)> 0)[0] + 1
print(len(train_y_quake))
print(len(train_y))
for idx in train_y_quake:
train_y.drop([idx],inplace=True)
 train_X.drop([idx],inplace = True)
 train_X.to_csv('train_features.csv', index=False)
 train_y.to_csv('train_y.csv', index=False)
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans_pca = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_pca'] = kmeans_pca.predict(train.values)
 test[f'clusters_pca'] = kmeans_pca.predict(test.values)
 train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )<feature_engineering>
scaler = StandardScaler()
scaler.fit(train_X)
 scaled_train_X = pd.DataFrame(scaler.transform(train_X), columns=train_X.columns)
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )<merge>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 test_X = pd.DataFrame(columns=train_X.columns, dtype=np.float64, index=submission.index )
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1)
 train = train_features.merge(train_targets_nonscored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_nonscored.columns]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
 target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
train = pd.get_dummies(train, columns=['cp_time','cp_dose'])
 test_ = pd.get_dummies(test, columns=['cp_time','cp_dose'])
 feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id']]<normalization>
for seg_id in tqdm_notebook(test_X.index):
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 seg.acoustic_data = seg.acoustic_data - mean_acoustic
create_features(seg_id, seg, test_X )
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.leaky_relu(self.dense1(x), 1e-3)
 x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<define_variables>
test_X.to_csv('test_features.csv', index=False )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 26
BATCH_SIZE = 256
LEARNING_RATE = 6e-4
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048<split>
scaled_test_X = pd.DataFrame(scaler.transform(test_X), columns=test_X.columns)
 scaled_test_X.values[1117]
def run_training(fold, seed):
seed_everything(seed)
 mskf = MultilabelStratifiedKFold(n_splits=7,random_state=seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
train.loc[v_idx, 'kfold'] = int(f)
 train['kfold'] = train['kfold'].astype(int)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"SEED: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed} ,FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"SEED{seed}_FOLD{fold}_nonscored.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"SEED{seed}_FOLD{fold}_nonscored.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<categorify>
n_fold = 5
def mae_cv(model):
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42 ).get_n_splits(scaled_train_X.values)
 mae = -cross_val_score(model, scaled_train_X.values, train_y, scoring="neg_mean_absolute_error",
verbose=0,
cv=folds)
 return mae
SEED = [77, 777]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test_[target_cols] = predictions<merge>
%%time
lgb_params = {'num_leaves': 51,
'min_data_in_leaf': 10,
'objective':'regression',
'max_depth': -1,
'learning_rate': 0.001,
"boosting": "gbdt",
"feature_fraction": 0.91,
"bagging_freq": 1,
"bagging_fraction": 0.91,
"bagging_seed": 42,
"metric": 'mae',
"lambda_l1": 0.1,
"verbosity": -1,
"nthread": -1,
"random_state": 42}
lgb_model = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.01, n_estimators=720,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11, n_jobs = -1)
 score = mae_cv(lgb_model)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
lgb_model
train = train.merge(train_targets_scored, on='sig_id')
 target = train[train_targets_scored.columns]
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id','kfold']]<define_variables>
lgb_gamma_model = lgb.LGBMRegressor(objective='gamma',num_leaves=5,
learning_rate=0.01, n_estimators=720,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11, n_jobs = -1)
 score = mae_cv(lgb_gamma_model)
 print("LGBM - gamma score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
lgb_gamma_model
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 26
BATCH_SIZE = 256
LEARNING_RATE = 6e-4
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048<split>
%%time
xgb_params = {'eta': 0.03,
'max_depth': 9,
'subsample': 0.85,
'objective': 'reg:linear',
'eval_metric': 'mae',
'silent': True,
'nthread': 4}
xgb_model = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1, eval_metric = 'mae',)
 score = mae_cv(xgb_model)
 print("XGB score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
xgb_model
def run_training(fold, seed):
seed_everything(seed)
 mskf = MultilabelStratifiedKFold(n_splits=7,random_state=seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
train.loc[v_idx, 'kfold'] = int(f)
 train['kfold'] = train['kfold'].astype(int)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"SEED: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed} ,FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"SEED{seed}_FOLD{fold}_scored.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"SEED{seed}_FOLD{fold}_scored.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<define_variables>
%%time
rf_model = RandomForestRegressor(n_estimators=120, n_jobs=-1, min_samples_leaf=1,
max_features = "auto",max_depth=15,)
 score = mae_cv(rf_model)
 print("Random Forest score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
rf_model
SEED = [77, 777]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions<save_to_csv>
%%time
params = {'loss_function':'MAE',}
cat_model = CatBoostRegressor(iterations=1000, eval_metric='MAE', verbose=False, **params)
 score = mae_cv(cat_model)
 print("Cat Boost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
cat_model
train[['sig_id']+target_cols].to_csv('oof9.csv', index=False)
 valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / len(target_cols)
 print("CV log_loss: ", score )<save_to_csv>
%%time
KRR_model = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
 score = mae_cv(KRR_model)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
print(score)
 KRR_model
public_id = list(df['sig_id'].values)
 test_id = list(test_features['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=target_cols)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_submit.loc[test.sig_id,:] = test[target_cols].values
df_submit.loc[test_features[test_features.cp_type=='ctl_vehicle'].sig_id]= 0
df_submit.to_csv('submission9.csv',index=True )<install_modules>
%%time
ENet_model = ElasticNet(alpha=0.0005, l1_ratio=0.9, random_state=3,max_iter=5000)
 score = mae_cv(ENet_model)
 print("Elastic Net score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
ENet_model
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet<set_options>
%%time
lasso_model = Lasso(alpha =0.0005, random_state=1)
 score = mae_cv(lasso_model)
 print("Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
lasso_model
warnings.filterwarnings("ignore")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1' 
<load_from_csv>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 df = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train_features2=train_features.copy()
test_features2=test_features.copy()<statistical_test>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
print(type(X))
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 print(KFold)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X.iloc[train_index], y.iloc[train_index])
 y_pred = instance.predict(X.iloc[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
def rank_gauss(data):
epsilon = 1e-6
for k in tqdm(GENES + CELLS):
r_cpu = data.loc[:,k].argsort().argsort()
r_cpu =(r_cpu/r_cpu.max() -0.5)*2
r_cpu = np.clip(r_cpu,-1+epsilon,1-epsilon)
 r_cpu = sp_erfinv(r_cpu)
 data.loc[:,k] = r_cpu * np.sqrt(2)
 return data
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
train_features = rank_gauss(train_features)
 test_features = rank_gauss(test_features )<set_options>
class StackingCVRegressorRetrained(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, regressors, meta_regressor, n_folds=5, use_features_in_secondary=False):
self.regressors = regressors
self.meta_regressor = meta_regressor
self.n_folds = n_folds
self.use_features_in_secondary = use_features_in_secondary
def fit(self, X, y):
self.regr_ = [clone(x)for x in self.regressors]
self.meta_regr_ = clone(self.meta_regressor)
 kfold = KFold(n_splits=self.n_folds, shuffle=True)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.regressors)))
 for i, regr in enumerate(self.regr_):
for train_idx, holdout_idx in kfold.split(X, y):
instance = clone(regr)
 instance.fit(X[train_idx], y[train_idx])
 out_of_fold_predictions[holdout_idx, i] = instance.predict(X[holdout_idx])
 if self.use_features_in_secondary:
self.meta_regr_.fit(np.hstack(( X, out_of_fold_predictions)) , y)
 else:
self.meta_regr_.fit(out_of_fold_predictions, y)
 for regr in self.regr_:
regr.fit(X, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
regr.predict(X)for regr in self.regr_
])
 if self.use_features_in_secondary:
return self.meta_regr_.predict(np.hstack(( X, meta_features)))
 else:
return self.meta_regr_.predict(meta_features)
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )<create_dataframe>
%%time
averaged_models = AveragingModels(models =(rf_model,cat_model))
score =mae_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
n_comp = 600
gpca= load(open('.. /input/moa-tabnet-correct-rankgauss-svd/gpca.pkl', 'rb'))
train2=(gpca.transform(train_features[GENES]))
test2 =(gpca.transform(test_features[GENES]))
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1)
 n_comp = 50
cpca= load(open('.. /input/moa-tabnet-correct-rankgauss-svd/cpca.pkl', 'rb'))
train2=(cpca.transform(train_features[CELLS]))
test2 =(cpca.transform(test_features[CELLS]))
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1)
 n_comp = 450
gsvd= load(open('.. /input/moa-tabnet-correct-rankgauss-svd/gsvd.pkl', 'rb'))
train2=(gsvd.transform(train_features[GENES]))
test2 =(gsvd.transform(test_features[GENES]))
train_gsvd = pd.DataFrame(train2, columns=[f'svd_G-{i}' for i in range(n_comp)])
 test_gsvd = pd.DataFrame(test2, columns=[f'svd_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gsvd), axis=1)
 test_features = pd.concat(( test_features, test_gsvd), axis=1)
 n_comp = 2
csvd= load(open('.. /input/moa-tabnet-correct-rankgauss-svd/csvd.pkl', 'rb'))
train2=(csvd.transform(train_features[CELLS]))
test2 =(csvd.transform(test_features[CELLS]))
train_csvd = pd.DataFrame(train2, columns=[f'svd_C-{i}' for i in range(n_comp)])
 test_csvd = pd.DataFrame(test2, columns=[f'svd_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_csvd), axis=1)
 test_features = pd.concat(( test_features, test_csvd), axis=1 )<concatenate>
averaged_models.fit(scaled_train_X.values, train_y)
 averaged2_train_predict = averaged_models.predict(scaled_train_X.values)
 print(mean_absolute_error(train_y, averaged2_train_predict))
c_n = [f for f in list(train_features.columns)if f not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train_features[c_n].var() >= 0.85 ).values
tmp = train_features[c_n].loc[:, mask]
train_features = pd.concat([train_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 tmp = test_features[c_n].loc[:, mask]
test_features = pd.concat([test_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1 )<find_best_model_class>
averaged_prediction = np.zeros(len(scaled_test_X))
averaged_prediction += averaged_models.predict(scaled_test_X.values)
 averaged_prediction
def fe_cluster_genes(train, test, n_clusters_g = 22, SEED = 42):
features_g = GENES
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
kmeans_genes = load(open('.. /input/moa-tabnet-correct-rankgauss-svd/kmeans_genes.pkl', 'rb'))
train[f'clusters_{kind}'] = kmeans_genes.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_genes.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 return train, test
train_features2 ,test_features2=fe_cluster_genes(train_features2,test_features2 )<find_best_model_class>
%%time
stacked_predict = StackingAveragedModels(base_models =(rf_model, xgb_model, lgb_model, cat_model,ENet_model),
meta_model =lasso_model)
 stacked_predict.fit(scaled_train_X, train_y)
def fe_cluster_cells(train, test, n_clusters_c = 4, SEED = 42):
features_c = CELLS
def create_cluster(train, test, features, kind = 'c', n_clusters = n_clusters_c):
train_ = train[features].copy()
test_ = test[features].copy()
kmeans_cells = load(open('.. /input/moa-tabnet-correct-rankgauss-svd/kmeans_cells.pkl', 'rb'))
train[f'clusters_{kind}'] = kmeans_cells.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_cells.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster_cells(train_features2,test_features2 )<concatenate>
stacked_train_pred = stacked_predict.predict(scaled_train_X)
 print(mean_absolute_error(train_y, stacked_train_pred))
stacked_prediction = np.zeros(len(scaled_test_X))
stacked_prediction += stacked_predict.predict(scaled_test_X)**1.0
stacked_prediction[0:4]
<find_best_model_class><EOS> 
submission.time_to_failure = averaged_prediction
submission.to_csv('submissionV30_averaged_cat_rf.csv',index=True)
 submission.time_to_failure = stacked_prediction
submission.to_csv('submissionV30_stacked.csv',index=True )