Kaggle competition: House Prices - Advanced Regression Techniques
if not TRAINING:
janestreet.competition.make_env.__called__ = False
env = janestreet.make_env()
th = 0.5
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
f0 = test_df['feature_0'].item()
f64 = test_df['feature_64'].item()
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt,training=False ).numpy() for model in models],axis=0 ).squeeze()
tb = time_bias(f0,f64)
 pred = np.mean(pred)* tb
pred_df.action = np.where(pred > th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<set_options>
df=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv" )
%matplotlib inline<load_from_csv>
df.skew().index[df.skew().values>1]
rows = 150_000
n_rows = 100*rows
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64} )<prepare_x_and_y>
outlier_cols=df.skew().index[df.skew().values>1]
rows = 150000
rowjump = 10000
njumps = int(15)
 segments = int(np.floor(train.shape[0] / rows))
X_train = pd.DataFrame(index=range(segments), dtype=np.float64,
columns=['ave', 'std', 'max', 'min',
'dave', 'dstd', 'dmax', 'dmin',
'q01', 'q02', 'q03', 'q04', 'q05', 'q06', 'q07', 'q08', 'q09', 'q10',
'q90', 'q91', 'q92', 'q93', 'q94', 'q95', 'q96', 'q97', 'q98', 'q99',
'dq01', 'dq02', 'dq03', 'dq04', 'dq05', 'dq06', 'dq07', 'dq08', 'dq09', 'dq10',
'dq90', 'dq91', 'dq92', 'dq93', 'dq94', 'dq95', 'dq96', 'dq97', 'dq98', 'dq99',
'kurt', 'skew',
'dkurt', 'dskew',
'ave_roll_std_10', 'std_roll_std_10', 'max_roll_std_10', 'min_roll_std_10',
'ave_roll_ave_10', 'std_roll_ave_10', 'max_roll_ave_10', 'min_roll_ave_10',
'ave_roll_std_100', 'std_roll_std_100', 'max_roll_std_100', 'min_roll_std_100',
'ave_roll_ave_100', 'std_roll_ave_100', 'max_roll_ave_100', 'min_roll_ave_100',
'ave_roll_std_1000', 'std_roll_std_1000', 'max_roll_std_1000', 'min_roll_std_1000',
'ave_roll_ave_1000', 'std_roll_ave_1000', 'max_roll_ave_1000', 'min_roll_ave_1000',
'ave_roll_ave_10', 'std_roll_ave_10', 'max_roll_ave_10', 'min_roll_ave_10',
'ave_roll_std_100', 'std_roll_std_100', 'max_roll_std_100', 'min_roll_std_100',
'ave_roll_ave_100', 'std_roll_ave_100', 'max_roll_ave_100', 'min_roll_ave_100',
'ave_roll_std_1000', 'std_roll_std_1000', 'max_roll_std_1000', 'min_roll_std_1000',
'ave_roll_ave_1000', 'std_roll_ave_1000', 'max_roll_ave_1000', 'min_roll_ave_1000',
'q01_roll_std_10', 'q02_roll_std_10', 'q03_roll_std_10', 'q04_roll_std_10',
'q05_roll_std_10', 'q06_roll_std_10', 'q07_roll_std_10', 'q08_roll_std_10',
'q09_roll_std_10', 'q10_roll_std_10',
'q90_roll_std_10', 'q91_roll_std_10', 'q92_roll_std_10', 'q93_roll_std_10',
'q94_roll_std_10', 'q95_roll_std_10', 'q96_roll_std_10', 'q97_roll_std_10',
'q98_roll_std_10', 'q99_roll_std_10',
'q01_roll_std_100', 'q02_roll_std_100', 'q03_roll_std_100', 'q04_roll_std_100',
'q05_roll_std_100', 'q06_roll_std_100', 'q07_roll_std_100', 'q08_roll_std_100',
'q09_roll_std_100', 'q10_roll_std_100',
'q90_roll_std_100', 'q91_roll_std_100', 'q92_roll_std_100', 'q93_roll_std_100',
'q94_roll_std_100', 'q95_roll_std_100', 'q96_roll_std_100', 'q97_roll_std_100',
'q98_roll_std_100', 'q99_roll_std_100',
'q01_roll_std_1000', 'q02_roll_std_1000', 'q03_roll_std_1000', 'q04_roll_std_1000',
'q05_roll_std_1000', 'q06_roll_std_1000', 'q07_roll_std_1000', 'q08_roll_std_1000',
'q09_roll_std_1000', 'q10_roll_std_1000',
'q90_roll_std_1000', 'q91_roll_std_1000', 'q92_roll_std_1000', 'q93_roll_std_1000',
'q94_roll_std_1000', 'q95_roll_std_1000', 'q96_roll_std_1000', 'q97_roll_std_1000',
'q98_roll_std_1000', 'q99_roll_std_1000',
'av_change_abs_roll_std_10', 'av_change_rate_roll_std_10',
'av_change_abs_roll_ave_10', 'av_change_rate_roll_ave_10',
'av_change_abs_roll_std_100', 'av_change_rate_roll_std_100',
'av_change_abs_roll_ave_100', 'av_change_rate_roll_ave_100',
'av_change_abs_roll_std_1000', 'av_change_rate_roll_std_1000',
'av_change_abs_roll_ave_1000', 'av_change_rate_roll_ave_1000',
'sum',
'tq01', 'tq02', 'tq03', 'tq04', 'tq05', 'tq06', 'tq07', 'tq08', 'tq09', 'tq10',
'tq90', 'tq91', 'tq92', 'tq93', 'tq94', 'tq95', 'tq96', 'tq97', 'tq98', 'tq99',
'dtq01', 'dtq02', 'dtq03', 'dtq04', 'dtq05', 'dtq06', 'dtq07', 'dtq08', 'dtq09', 'dtq10',
'dtq90', 'dtq91', 'dtq92', 'dtq93', 'dtq94', 'dtq95', 'dtq96', 'dtq97', 'dtq98', 'dtq99'])
 y_train_raw = pd.DataFrame(index=range(segments), dtype=np.float64,
columns=['time_to_failure'])
 for njump in range(njumps):
for segment in tqdm(range(segments)) :
seg = train.iloc[segment*rows+njump*rowjump:segment*rows+rows+njump*rowjump]
x_raw = seg['acoustic_data']
x = seg['acoustic_data'].values
dx = np.diff(x)
 y = seg['time_to_failure'].values[-1]
y_train_raw.loc[segment+njump*segments, 'time_to_failure'] = y
X_train.loc[segment+njump*segments, 'ave'] = x.mean()
X_train.loc[segment+njump*segments, 'std'] = x.std()
X_train.loc[segment+njump*segments, 'max'] = x.max()
X_train.loc[segment+njump*segments, 'min'] = x.min()
X_train.loc[segment+njump*segments, 'dave'] = dx.mean()
X_train.loc[segment+njump*segments, 'dstd'] = dx.std()
X_train.loc[segment+njump*segments, 'dmax'] = dx.max()
X_train.loc[segment+njump*segments, 'dmin'] = dx.min()
X_train.loc[segment+njump*segments, 'q01'] = np.quantile(x,0.01)
 X_train.loc[segment+njump*segments, 'q02'] = np.quantile(x,0.02)
 X_train.loc[segment+njump*segments, 'q03'] = np.quantile(x,0.03)
 X_train.loc[segment+njump*segments, 'q04'] = np.quantile(x,0.04)
 X_train.loc[segment+njump*segments, 'q05'] = np.quantile(x,0.05)
 X_train.loc[segment+njump*segments, 'q06'] = np.quantile(x,0.06)
 X_train.loc[segment+njump*segments, 'q07'] = np.quantile(x,0.07)
 X_train.loc[segment+njump*segments, 'q08'] = np.quantile(x,0.08)
 X_train.loc[segment+njump*segments, 'q09'] = np.quantile(x,0.09)
 X_train.loc[segment+njump*segments, 'q10'] = np.quantile(x,0.10)
 X_train.loc[segment+njump*segments, 'q90'] = np.quantile(x,0.90)
 X_train.loc[segment+njump*segments, 'q91'] = np.quantile(x,0.91)
 X_train.loc[segment+njump*segments, 'q92'] = np.quantile(x,0.92)
 X_train.loc[segment+njump*segments, 'q93'] = np.quantile(x,0.93)
 X_train.loc[segment+njump*segments, 'q94'] = np.quantile(x,0.94)
 X_train.loc[segment+njump*segments, 'q95'] = np.quantile(x,0.95)
 X_train.loc[segment+njump*segments, 'q96'] = np.quantile(x,0.96)
 X_train.loc[segment+njump*segments, 'q97'] = np.quantile(x,0.97)
 X_train.loc[segment+njump*segments, 'q98'] = np.quantile(x,0.98)
 X_train.loc[segment+njump*segments, 'q99'] = np.quantile(x,0.99)
 X_train.loc[segment+njump*segments, 'dq01'] = np.quantile(dx,0.01)
 X_train.loc[segment+njump*segments, 'dq02'] = np.quantile(dx,0.02)
 X_train.loc[segment+njump*segments, 'dq03'] = np.quantile(dx,0.03)
 X_train.loc[segment+njump*segments, 'dq04'] = np.quantile(dx,0.04)
 X_train.loc[segment+njump*segments, 'dq05'] = np.quantile(dx,0.05)
 X_train.loc[segment+njump*segments, 'dq06'] = np.quantile(dx,0.06)
 X_train.loc[segment+njump*segments, 'dq07'] = np.quantile(dx,0.07)
 X_train.loc[segment+njump*segments, 'dq08'] = np.quantile(dx,0.08)
 X_train.loc[segment+njump*segments, 'dq09'] = np.quantile(dx,0.09)
 X_train.loc[segment+njump*segments, 'dq10'] = np.quantile(dx,0.10)
 X_train.loc[segment+njump*segments, 'dq90'] = np.quantile(dx,0.90)
 X_train.loc[segment+njump*segments, 'dq91'] = np.quantile(dx,0.91)
 X_train.loc[segment+njump*segments, 'dq92'] = np.quantile(dx,0.92)
 X_train.loc[segment+njump*segments, 'dq93'] = np.quantile(dx,0.93)
 X_train.loc[segment+njump*segments, 'dq94'] = np.quantile(dx,0.94)
 X_train.loc[segment+njump*segments, 'dq95'] = np.quantile(dx,0.95)
 X_train.loc[segment+njump*segments, 'dq96'] = np.quantile(dx,0.96)
 X_train.loc[segment+njump*segments, 'dq97'] = np.quantile(dx,0.97)
 X_train.loc[segment+njump*segments, 'dq98'] = np.quantile(dx,0.98)
 X_train.loc[segment+njump*segments, 'dq99'] = np.quantile(dx,0.99)
 X_train.loc[segment+njump*segments, 'kurt'] = kurtosis(x)
 X_train.loc[segment+njump*segments, 'skew'] = skew(x)
 X_train.loc[segment+njump*segments, 'dkurt'] = kurtosis(dx)
 X_train.loc[segment+njump*segments, 'dskew'] = skew(dx)
 for windows in [10,100,1000]:
x_roll_std = x_raw.rolling(windows ).std().dropna().values
x_roll_mean = x_raw.rolling(windows ).mean().dropna().values
X_train.loc[segment+njump*segments, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X_train.loc[segment+njump*segments, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X_train.loc[segment+njump*segments, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X_train.loc[segment+njump*segments, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X_train.loc[segment+njump*segments, 'ave_roll_ave_' + str(windows)] = x_roll_mean.mean()
X_train.loc[segment+njump*segments, 'std_roll_ave_' + str(windows)] = x_roll_mean.std()
X_train.loc[segment+njump*segments, 'max_roll_ave_' + str(windows)] = x_roll_mean.max()
X_train.loc[segment+njump*segments, 'min_roll_ave_' + str(windows)] = x_roll_mean.min()
X_train.loc[segment+njump*segments, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.01)
 X_train.loc[segment+njump*segments, 'q02_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.02)
 X_train.loc[segment+njump*segments, 'q03_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.03)
 X_train.loc[segment+njump*segments, 'q04_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.04)
 X_train.loc[segment+njump*segments, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.05)
 X_train.loc[segment+njump*segments, 'q06_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.06)
 X_train.loc[segment+njump*segments, 'q07_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.07)
 X_train.loc[segment+njump*segments, 'q08_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.08)
 X_train.loc[segment+njump*segments, 'q09_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.09)
 X_train.loc[segment+njump*segments, 'q10_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.10)
 X_train.loc[segment+njump*segments, 'q90_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.90)
 X_train.loc[segment+njump*segments, 'q91_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.91)
 X_train.loc[segment+njump*segments, 'q92_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.92)
 X_train.loc[segment+njump*segments, 'q93_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.93)
 X_train.loc[segment+njump*segments, 'q94_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.94)
 X_train.loc[segment+njump*segments, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.95)
 X_train.loc[segment+njump*segments, 'q96_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.96)
 X_train.loc[segment+njump*segments, 'q97_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.97)
 X_train.loc[segment+njump*segments, 'q98_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.98)
 X_train.loc[segment+njump*segments, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.99)
 X_train.loc[segment+njump*segments, 'q01_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.01)
 X_train.loc[segment+njump*segments, 'q02_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.02)
 X_train.loc[segment+njump*segments, 'q03_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.03)
 X_train.loc[segment+njump*segments, 'q04_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.04)
 X_train.loc[segment+njump*segments, 'q05_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.05)
 X_train.loc[segment+njump*segments, 'q06_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.06)
 X_train.loc[segment+njump*segments, 'q07_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.07)
 X_train.loc[segment+njump*segments, 'q08_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.08)
 X_train.loc[segment+njump*segments, 'q09_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.09)
 X_train.loc[segment+njump*segments, 'q10_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.10)
 X_train.loc[segment+njump*segments, 'q90_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.90)
 X_train.loc[segment+njump*segments, 'q91_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.91)
 X_train.loc[segment+njump*segments, 'q92_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.92)
 X_train.loc[segment+njump*segments, 'q93_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.93)
 X_train.loc[segment+njump*segments, 'q94_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.94)
 X_train.loc[segment+njump*segments, 'q95_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.95)
 X_train.loc[segment+njump*segments, 'q96_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.96)
 X_train.loc[segment+njump*segments, 'q97_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.97)
 X_train.loc[segment+njump*segments, 'q98_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.98)
 X_train.loc[segment+njump*segments, 'q99_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.99)
 X_train.loc[segment+njump*segments, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X_train.loc[segment+njump*segments, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X_train.loc[segment+njump*segments, 'av_change_abs_roll_ave_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_train.loc[segment+njump*segments, 'av_change_rate_roll_ave_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X_train.loc[segment+njump*segments, 'sum'] = sum(abs(x))
X_train.loc[segment+njump*segments, 'tq01'] = np.count_nonzero(x < np.quantile(x,0.01))
X_train.loc[segment+njump*segments, 'tq02'] = np.count_nonzero(x < np.quantile(x,0.02))
X_train.loc[segment+njump*segments, 'tq03'] = np.count_nonzero(x > np.quantile(x,0.03))
X_train.loc[segment+njump*segments, 'tq04'] = np.count_nonzero(x > np.quantile(x,0.04))
X_train.loc[segment+njump*segments, 'tq05'] = np.count_nonzero(x < np.quantile(x,0.05))
X_train.loc[segment+njump*segments, 'tq06'] = np.count_nonzero(x < np.quantile(x,0.06))
X_train.loc[segment+njump*segments, 'tq07'] = np.count_nonzero(x > np.quantile(x,0.07))
X_train.loc[segment+njump*segments, 'tq08'] = np.count_nonzero(x > np.quantile(x,0.08))
X_train.loc[segment+njump*segments, 'tq09'] = np.count_nonzero(x < np.quantile(x,0.09))
X_train.loc[segment+njump*segments, 'tq10'] = np.count_nonzero(x < np.quantile(x,0.10))
X_train.loc[segment+njump*segments, 'tq90'] = np.count_nonzero(x < np.quantile(x,0.90))
X_train.loc[segment+njump*segments, 'tq91'] = np.count_nonzero(x > np.quantile(x,0.91))
X_train.loc[segment+njump*segments, 'tq92'] = np.count_nonzero(x > np.quantile(x,0.92))
X_train.loc[segment+njump*segments, 'tq93'] = np.count_nonzero(x < np.quantile(x,0.93))
X_train.loc[segment+njump*segments, 'tq94'] = np.count_nonzero(x < np.quantile(x,0.94))
X_train.loc[segment+njump*segments, 'tq95'] = np.count_nonzero(x > np.quantile(x,0.95))
X_train.loc[segment+njump*segments, 'tq96'] = np.count_nonzero(x > np.quantile(x,0.96))
X_train.loc[segment+njump*segments, 'tq97'] = np.count_nonzero(x < np.quantile(x,0.97))
X_train.loc[segment+njump*segments, 'tq98'] = np.count_nonzero(x < np.quantile(x,0.98))
X_train.loc[segment+njump*segments, 'tq99'] = np.count_nonzero(x > np.quantile(x,0.99))
X_train.loc[segment+njump*segments, 'dtq01'] = np.count_nonzero(x < np.quantile(dx,0.01))
X_train.loc[segment+njump*segments, 'dtq02'] = np.count_nonzero(x < np.quantile(dx,0.02))
X_train.loc[segment+njump*segments, 'dtq03'] = np.count_nonzero(x > np.quantile(dx,0.03))
X_train.loc[segment+njump*segments, 'dtq04'] = np.count_nonzero(x > np.quantile(dx,0.04))
X_train.loc[segment+njump*segments, 'dtq05'] = np.count_nonzero(x < np.quantile(dx,0.05))
X_train.loc[segment+njump*segments, 'dtq06'] = np.count_nonzero(x < np.quantile(dx,0.06))
X_train.loc[segment+njump*segments, 'dtq07'] = np.count_nonzero(x > np.quantile(dx,0.07))
X_train.loc[segment+njump*segments, 'dtq08'] = np.count_nonzero(x > np.quantile(dx,0.08))
X_train.loc[segment+njump*segments, 'dtq09'] = np.count_nonzero(x < np.quantile(dx,0.09))
X_train.loc[segment+njump*segments, 'dtq10'] = np.count_nonzero(x < np.quantile(dx,0.10))
X_train.loc[segment+njump*segments, 'dtq90'] = np.count_nonzero(x < np.quantile(dx,0.90))
X_train.loc[segment+njump*segments, 'dtq91'] = np.count_nonzero(x > np.quantile(dx,0.91))
X_train.loc[segment+njump*segments, 'dtq92'] = np.count_nonzero(x > np.quantile(dx,0.92))
X_train.loc[segment+njump*segments, 'dtq93'] = np.count_nonzero(x < np.quantile(dx,0.93))
X_train.loc[segment+njump*segments, 'dtq94'] = np.count_nonzero(x < np.quantile(dx,0.94))
X_train.loc[segment+njump*segments, 'dtq95'] = np.count_nonzero(x > np.quantile(dx,0.95))
X_train.loc[segment+njump*segments, 'dtq96'] = np.count_nonzero(x > np.quantile(dx,0.96))
X_train.loc[segment+njump*segments, 'dtq97'] = np.count_nonzero(x < np.quantile(dx,0.97))
X_train.loc[segment+njump*segments, 'dtq98'] = np.count_nonzero(x < np.quantile(dx,0.98))
X_train.loc[segment+njump*segments, 'dtq99'] = np.count_nonzero(x > np.quantile(dx,0.99)) 
<load_from_csv>
df.isnull().sum().index[df.isnull().sum().values>0]
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 X_submit = pd.DataFrame(index=submission.index, dtype=np.float64, columns=X_train.columns)
 for i, seg_id in enumerate(tqdm(X_submit.index)) :
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 x_raw = seg['acoustic_data']
x = seg['acoustic_data'].values
dx = np.diff(x)
 X_submit.loc[seg_id, 'ave'] = x.mean()
X_submit.loc[seg_id, 'std'] = x.std()
X_submit.loc[seg_id, 'max'] = x.max()
X_submit.loc[seg_id, 'min'] = x.min()
X_submit.loc[seg_id, 'dave'] = dx.mean()
X_submit.loc[seg_id, 'dstd'] = dx.std()
X_submit.loc[seg_id, 'dmax'] = dx.max()
X_submit.loc[seg_id, 'dmin'] = dx.min()
X_submit.loc[seg_id, 'q01'] = np.quantile(x,0.01)
 X_submit.loc[seg_id, 'q02'] = np.quantile(x,0.02)
 X_submit.loc[seg_id, 'q03'] = np.quantile(x,0.03)
 X_submit.loc[seg_id, 'q04'] = np.quantile(x,0.04)
 X_submit.loc[seg_id, 'q05'] = np.quantile(x,0.05)
 X_submit.loc[seg_id, 'q06'] = np.quantile(x,0.06)
 X_submit.loc[seg_id, 'q07'] = np.quantile(x,0.07)
 X_submit.loc[seg_id, 'q08'] = np.quantile(x,0.08)
 X_submit.loc[seg_id, 'q09'] = np.quantile(x,0.09)
 X_submit.loc[seg_id, 'q10'] = np.quantile(x,0.10)
 X_submit.loc[seg_id, 'q90'] = np.quantile(x,0.90)
 X_submit.loc[seg_id, 'q91'] = np.quantile(x,0.91)
 X_submit.loc[seg_id, 'q92'] = np.quantile(x,0.92)
 X_submit.loc[seg_id, 'q93'] = np.quantile(x,0.93)
 X_submit.loc[seg_id, 'q94'] = np.quantile(x,0.94)
 X_submit.loc[seg_id, 'q95'] = np.quantile(x,0.95)
 X_submit.loc[seg_id, 'q96'] = np.quantile(x,0.96)
 X_submit.loc[seg_id, 'q97'] = np.quantile(x,0.97)
 X_submit.loc[seg_id, 'q98'] = np.quantile(x,0.98)
 X_submit.loc[seg_id, 'q99'] = np.quantile(x,0.99)
 X_submit.loc[seg_id, 'dq01'] = np.quantile(dx,0.01)
 X_submit.loc[seg_id, 'dq02'] = np.quantile(dx,0.02)
 X_submit.loc[seg_id, 'dq03'] = np.quantile(dx,0.03)
 X_submit.loc[seg_id, 'dq04'] = np.quantile(dx,0.04)
 X_submit.loc[seg_id, 'dq05'] = np.quantile(dx,0.05)
 X_submit.loc[seg_id, 'dq06'] = np.quantile(dx,0.06)
 X_submit.loc[seg_id, 'dq07'] = np.quantile(dx,0.07)
 X_submit.loc[seg_id, 'dq08'] = np.quantile(dx,0.08)
 X_submit.loc[seg_id, 'dq09'] = np.quantile(dx,0.09)
 X_submit.loc[seg_id, 'dq10'] = np.quantile(dx,0.10)
 X_submit.loc[seg_id, 'dq90'] = np.quantile(dx,0.90)
 X_submit.loc[seg_id, 'dq91'] = np.quantile(dx,0.91)
 X_submit.loc[seg_id, 'dq92'] = np.quantile(dx,0.92)
 X_submit.loc[seg_id, 'dq93'] = np.quantile(dx,0.93)
 X_submit.loc[seg_id, 'dq94'] = np.quantile(dx,0.94)
 X_submit.loc[seg_id, 'dq95'] = np.quantile(dx,0.95)
 X_submit.loc[seg_id, 'dq96'] = np.quantile(dx,0.96)
 X_submit.loc[seg_id, 'dq97'] = np.quantile(dx,0.97)
 X_submit.loc[seg_id, 'dq98'] = np.quantile(dx,0.98)
 X_submit.loc[seg_id, 'dq99'] = np.quantile(dx,0.99)
 X_submit.loc[seg_id, 'kurt'] = kurtosis(x)
 X_submit.loc[seg_id, 'skew'] = skew(x)
 X_submit.loc[seg_id, 'dkurt'] = kurtosis(dx)
 X_submit.loc[seg_id, 'dskew'] = skew(dx)
 for windows in [10,100,1000]:
x_roll_std = x_raw.rolling(windows ).std().dropna().values
x_roll_mean = x_raw.rolling(windows ).mean().dropna().values
X_submit.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X_submit.loc[seg_id, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X_submit.loc[seg_id, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X_submit.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X_submit.loc[seg_id, 'ave_roll_ave_' + str(windows)] = x_roll_mean.mean()
X_submit.loc[seg_id, 'std_roll_ave_' + str(windows)] = x_roll_mean.std()
X_submit.loc[seg_id, 'max_roll_ave_' + str(windows)] = x_roll_mean.max()
X_submit.loc[seg_id, 'min_roll_ave_' + str(windows)] = x_roll_mean.min()
X_submit.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.01)
 X_submit.loc[seg_id, 'q02_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.02)
 X_submit.loc[seg_id, 'q03_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.03)
 X_submit.loc[seg_id, 'q04_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.04)
 X_submit.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.05)
 X_submit.loc[seg_id, 'q06_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.06)
 X_submit.loc[seg_id, 'q07_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.07)
 X_submit.loc[seg_id, 'q08_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.08)
 X_submit.loc[seg_id, 'q09_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.09)
 X_submit.loc[seg_id, 'q10_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.10)
 X_submit.loc[seg_id, 'q90_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.90)
 X_submit.loc[seg_id, 'q91_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.91)
 X_submit.loc[seg_id, 'q92_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.92)
 X_submit.loc[seg_id, 'q93_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.93)
 X_submit.loc[seg_id, 'q94_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.94)
 X_submit.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.95)
 X_submit.loc[seg_id, 'q96_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.96)
 X_submit.loc[seg_id, 'q97_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.97)
 X_submit.loc[seg_id, 'q98_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.98)
 X_submit.loc[seg_id, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std,0.99)
 X_submit.loc[seg_id, 'q01_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.01)
 X_submit.loc[seg_id, 'q02_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.02)
 X_submit.loc[seg_id, 'q03_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.03)
 X_submit.loc[seg_id, 'q04_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.04)
 X_submit.loc[seg_id, 'q05_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.05)
 X_submit.loc[seg_id, 'q06_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.06)
 X_submit.loc[seg_id, 'q07_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.07)
 X_submit.loc[seg_id, 'q08_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.08)
 X_submit.loc[seg_id, 'q09_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.09)
 X_submit.loc[seg_id, 'q10_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.10)
 X_submit.loc[seg_id, 'q90_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.90)
 X_submit.loc[seg_id, 'q91_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.91)
 X_submit.loc[seg_id, 'q92_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.92)
 X_submit.loc[seg_id, 'q93_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.93)
 X_submit.loc[seg_id, 'q94_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.94)
 X_submit.loc[seg_id, 'q95_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.95)
 X_submit.loc[seg_id, 'q96_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.96)
 X_submit.loc[seg_id, 'q97_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.97)
 X_submit.loc[seg_id, 'q98_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.98)
 X_submit.loc[seg_id, 'q99_roll_ave_' + str(windows)] = np.quantile(x_roll_mean,0.99)
 X_submit.loc[seg_id, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X_submit.loc[seg_id, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X_submit.loc[seg_id, 'av_change_abs_roll_ave_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_submit.loc[seg_id, 'av_change_rate_roll_ave_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X_submit.loc[seg_id, 'sum'] = sum(abs(x))
X_submit.loc[seg_id, 'tq01'] = np.count_nonzero(x < np.quantile(x,0.01))
X_submit.loc[seg_id, 'tq02'] = np.count_nonzero(x < np.quantile(x,0.02))
X_submit.loc[seg_id, 'tq03'] = np.count_nonzero(x > np.quantile(x,0.03))
X_submit.loc[seg_id, 'tq04'] = np.count_nonzero(x > np.quantile(x,0.04))
X_submit.loc[seg_id, 'tq05'] = np.count_nonzero(x < np.quantile(x,0.05))
X_submit.loc[seg_id, 'tq06'] = np.count_nonzero(x < np.quantile(x,0.06))
X_submit.loc[seg_id, 'tq07'] = np.count_nonzero(x > np.quantile(x,0.07))
X_submit.loc[seg_id, 'tq08'] = np.count_nonzero(x > np.quantile(x,0.08))
X_submit.loc[seg_id, 'tq09'] = np.count_nonzero(x < np.quantile(x,0.09))
X_submit.loc[seg_id, 'tq10'] = np.count_nonzero(x < np.quantile(x,0.10))
X_submit.loc[seg_id, 'tq90'] = np.count_nonzero(x < np.quantile(x,0.90))
X_submit.loc[seg_id, 'tq91'] = np.count_nonzero(x > np.quantile(x,0.91))
X_submit.loc[seg_id, 'tq92'] = np.count_nonzero(x > np.quantile(x,0.92))
X_submit.loc[seg_id, 'tq93'] = np.count_nonzero(x < np.quantile(x,0.93))
X_submit.loc[seg_id, 'tq94'] = np.count_nonzero(x < np.quantile(x,0.94))
X_submit.loc[seg_id, 'tq95'] = np.count_nonzero(x > np.quantile(x,0.95))
X_submit.loc[seg_id, 'tq96'] = np.count_nonzero(x > np.quantile(x,0.96))
X_submit.loc[seg_id, 'tq97'] = np.count_nonzero(x < np.quantile(x,0.97))
X_submit.loc[seg_id, 'tq98'] = np.count_nonzero(x < np.quantile(x,0.98))
X_submit.loc[seg_id, 'tq99'] = np.count_nonzero(x > np.quantile(x,0.99))
X_submit.loc[seg_id, 'dtq01'] = np.count_nonzero(x < np.quantile(dx,0.01))
X_submit.loc[seg_id, 'dtq02'] = np.count_nonzero(x < np.quantile(dx,0.02))
X_submit.loc[seg_id, 'dtq03'] = np.count_nonzero(x > np.quantile(dx,0.03))
X_submit.loc[seg_id, 'dtq04'] = np.count_nonzero(x > np.quantile(dx,0.04))
X_submit.loc[seg_id, 'dtq05'] = np.count_nonzero(x < np.quantile(dx,0.05))
X_submit.loc[seg_id, 'dtq06'] = np.count_nonzero(x < np.quantile(dx,0.06))
X_submit.loc[seg_id, 'dtq07'] = np.count_nonzero(x > np.quantile(dx,0.07))
X_submit.loc[seg_id, 'dtq08'] = np.count_nonzero(x > np.quantile(dx,0.08))
X_submit.loc[seg_id, 'dtq09'] = np.count_nonzero(x < np.quantile(dx,0.09))
X_submit.loc[seg_id, 'dtq10'] = np.count_nonzero(x < np.quantile(dx,0.10))
X_submit.loc[seg_id, 'dtq90'] = np.count_nonzero(x < np.quantile(dx,0.90))
X_submit.loc[seg_id, 'dtq91'] = np.count_nonzero(x > np.quantile(dx,0.91))
X_submit.loc[seg_id, 'dtq92'] = np.count_nonzero(x > np.quantile(dx,0.92))
X_submit.loc[seg_id, 'dtq93'] = np.count_nonzero(x < np.quantile(dx,0.93))
X_submit.loc[seg_id, 'dtq94'] = np.count_nonzero(x < np.quantile(dx,0.94))
X_submit.loc[seg_id, 'dtq95'] = np.count_nonzero(x > np.quantile(dx,0.95))
X_submit.loc[seg_id, 'dtq96'] = np.count_nonzero(x > np.quantile(dx,0.96))
X_submit.loc[seg_id, 'dtq97'] = np.count_nonzero(x < np.quantile(dx,0.97))
X_submit.loc[seg_id, 'dtq98'] = np.count_nonzero(x < np.quantile(dx,0.98))
X_submit.loc[seg_id, 'dtq99'] = np.count_nonzero(x > np.quantile(dx,0.99)) 
<count_values>
df_copy=df.copy()
multiprocessing.cpu_count()<init_hyperparams>
cat_columns=df_copy.select_dtypes(include=['O','object'] ).columns
for cols in cat_columns:
df_copy[cols].fillna(df_copy[cols].mode() [0],inplace=True )
params = {'boosting_type': 'gbdt',
'objective': 'regression_l1',
'nthread': 4,
'num_leaves': 64,
'learning_rate': 0.05,
'max_bin': 512,
'colsample_bytree': 1,
'reg_alpha': 0.1,
'reg_lambda': 0.1,
'min_split_gain': 0.1,
'min_child_weight': 0.1,
'min_child_samples': 20,
'metric' : 'mae'}
gbm = lgb.LGBMRegressor(boosting_type= 'gbdt',
objective = 'regression_l1',
n_jobs = 4,
silent = True,
max_bin = params['max_bin'],
min_split_gain = params['min_split_gain'],
min_child_weight = params['min_child_weight'],
min_child_samples = params['min_child_samples'])
 gbm.get_params().keys()<train_model>
num_columns=df_copy.select_dtypes(exclude=['O','object'] ).columns
for cols in num_columns:
if cols in outlier_cols:
df_copy[cols].fillna(df_copy[cols].median() ,inplace=True)
 else:
df_copy[cols].fillna(df_copy[cols].mean() ,inplace=True )
print('Starting training...')
 folds = KFold(n_splits=5, shuffle=True, random_state=11)
 for fold_n,(train_index, valid_index)in enumerate(folds.split(X_train)) :
print('Fold', fold_n)
 X_train_f, X_valid = X_train.iloc[train_index], X_train.iloc[valid_index]
y_train_f, y_valid = y_train_raw.iloc[train_index], y_train_raw.iloc[valid_index]
gbm.fit(X_train_f, y_train_f.values.flatten() ,
eval_set=[(X_valid, y_valid.values.flatten())],
eval_metric='mae',
early_stopping_rounds=200)
 print('Starting predicting...')
 y_pred = gbm.predict(X_valid, num_iteration=gbm.best_iteration_)
 print('The mae of prediction is:', mean_absolute_error(y_valid.values.flatten() , y_pred))
print('Feature importances:', list(gbm.feature_importances_))<predict_on_test>
df_copy.isnull().sum().index[df_copy.isnull().sum().values>0]
y_submit = gbm.predict(X_submit )<save_to_csv>
fig=px.box(df_copy['LotArea'])
 fig.show("notebook" )
submission['time_to_failure'] = y_submit
print(submission.head())
 submission.to_csv('submission.csv' )<set_options>
df_copy.corr() ['SalePrice'].sort_values()
warnings.filterwarnings("ignore" )<load_from_csv>
X=df_copy.copy()
X.drop(["SalePrice","Id"],axis=1,inplace=True)
 y=df_copy['SalePrice']
%%time
train_df = pd.read_csv(os.path.join(PATH,'train.csv'), dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32} )<train_model>
y = np.log1p(y )
print("Train: rows:{} cols:{}".format(train_df.shape[0], train_df.shape[1]))<define_variables>
num_train_columns=['MSSubClass', 'LotFrontage', 'LotArea', 'OverallQual',
'OverallCond', 'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1',
'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF',
'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath',
'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd',
'Fireplaces', 'GarageYrBlt', 'GarageCars', 'GarageArea', 'WoodDeckSF',
'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea',
'MiscVal', 'MoSold', 'YrSold']
rows = 150000
segments = int(np.floor(train_df.shape[0] / rows))
print("Number of segments: ", segments )<normalization>
qt=QuantileTransformer(output_distribution='normal',random_state=0)
 X_num_transformed=qt.fit_transform(X[num_train_columns])
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def classic_sta_lta(x, length_sta, length_lta):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]
sta /= length_sta
lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]
lta /= length_lta
sta[:length_lta - 1] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta<prepare_x_and_y>
df_num_transformed=pd.DataFrame(X_num_transformed.reshape(-1,36),columns=X[num_train_columns].columns )
train_X = pd.DataFrame(index=range(segments), dtype=np.float64)
 train_y = pd.DataFrame(index=range(segments), dtype=np.float64, columns=['time_to_failure'])
 total_mean = train_df['acoustic_data'].mean()
total_std = train_df['acoustic_data'].std()
total_max = train_df['acoustic_data'].max()
total_min = train_df['acoustic_data'].min()
total_sum = train_df['acoustic_data'].sum()
total_abs_sum = np.abs(train_df['acoustic_data'] ).sum()<feature_engineering>
rs=RobustScaler()
X_num_transformed=rs.fit_transform(df_num_transformed )
def create_features(seg_id, seg, X):
xc = pd.Series(seg['acoustic_data'].values)
 zc = np.fft.fft(xc)
 X.loc[seg_id, 'mean'] = xc.mean()
X.loc[seg_id, 'std'] = xc.std()
X.loc[seg_id, 'max'] = xc.max()
X.loc[seg_id, 'min'] = xc.min()
realFFT = np.real(zc)
 imagFFT = np.imag(zc)
 X.loc[seg_id, 'Rmean'] = realFFT.mean()
X.loc[seg_id, 'Rstd'] = realFFT.std()
X.loc[seg_id, 'Rmax'] = realFFT.max()
X.loc[seg_id, 'Rmin'] = realFFT.min()
X.loc[seg_id, 'Imean'] = imagFFT.mean()
X.loc[seg_id, 'Istd'] = imagFFT.std()
X.loc[seg_id, 'Imax'] = imagFFT.max()
X.loc[seg_id, 'Imin'] = imagFFT.min()
X.loc[seg_id, 'Rmean_last_5000'] = realFFT[-5000:].mean()
X.loc[seg_id, 'Rstd__last_5000'] = realFFT[-5000:].std()
X.loc[seg_id, 'Rmax_last_5000'] = realFFT[-5000:].max()
X.loc[seg_id, 'Rmin_last_5000'] = realFFT[-5000:].min()
X.loc[seg_id, 'Rmean_last_15000'] = realFFT[-15000:].mean()
X.loc[seg_id, 'Rstd_last_15000'] = realFFT[-15000:].std()
X.loc[seg_id, 'Rmax_last_15000'] = realFFT[-15000:].max()
X.loc[seg_id, 'Rmin_last_15000'] = realFFT[-15000:].min()
X.loc[seg_id, 'mean_change_abs'] = np.mean(np.diff(xc))
X.loc[seg_id, 'mean_change_rate'] = np.mean(np.nonzero(( np.diff(xc)/ xc[:-1])) [0])
 X.loc[seg_id, 'abs_max'] = np.abs(xc ).max()
X.loc[seg_id, 'abs_min'] = np.abs(xc ).min()
X.loc[seg_id, 'std_first_50000'] = xc[:50000].std()
X.loc[seg_id, 'std_last_50000'] = xc[-50000:].std()
X.loc[seg_id, 'std_first_10000'] = xc[:10000].std()
X.loc[seg_id, 'std_last_10000'] = xc[-10000:].std()
X.loc[seg_id, 'avg_first_50000'] = xc[:50000].mean()
X.loc[seg_id, 'avg_last_50000'] = xc[-50000:].mean()
X.loc[seg_id, 'avg_first_10000'] = xc[:10000].mean()
X.loc[seg_id, 'avg_last_10000'] = xc[-10000:].mean()
X.loc[seg_id, 'min_first_50000'] = xc[:50000].min()
X.loc[seg_id, 'min_last_50000'] = xc[-50000:].min()
X.loc[seg_id, 'min_first_10000'] = xc[:10000].min()
X.loc[seg_id, 'min_last_10000'] = xc[-10000:].min()
X.loc[seg_id, 'max_first_50000'] = xc[:50000].max()
X.loc[seg_id, 'max_last_50000'] = xc[-50000:].max()
X.loc[seg_id, 'max_first_10000'] = xc[:10000].max()
X.loc[seg_id, 'max_last_10000'] = xc[-10000:].max()
X.loc[seg_id, 'max_to_min'] = xc.max() / np.abs(xc.min())
 X.loc[seg_id, 'max_to_min_diff'] = xc.max() - np.abs(xc.min())
 X.loc[seg_id, 'count_big'] = len(xc[np.abs(xc)> 500])
 X.loc[seg_id, 'sum'] = xc.sum()
X.loc[seg_id, 'mean_change_rate_first_50000'] = np.mean(np.nonzero(( np.diff(xc[:50000])/ xc[:50000][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_last_50000'] = np.mean(np.nonzero(( np.diff(xc[-50000:])/ xc[-50000:][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_first_10000'] = np.mean(np.nonzero(( np.diff(xc[:10000])/ xc[:10000][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_last_10000'] = np.mean(np.nonzero(( np.diff(xc[-10000:])/ xc[-10000:][:-1])) [0])
 X.loc[seg_id, 'q95'] = np.quantile(xc, 0.95)
 X.loc[seg_id, 'q99'] = np.quantile(xc, 0.99)
 X.loc[seg_id, 'q05'] = np.quantile(xc, 0.05)
 X.loc[seg_id, 'q01'] = np.quantile(xc, 0.01)
 X.loc[seg_id, 'abs_q95'] = np.quantile(np.abs(xc), 0.95)
 X.loc[seg_id, 'abs_q99'] = np.quantile(np.abs(xc), 0.99)
 X.loc[seg_id, 'abs_q05'] = np.quantile(np.abs(xc), 0.05)
 X.loc[seg_id, 'abs_q01'] = np.quantile(np.abs(xc), 0.01)
 X.loc[seg_id, 'trend'] = add_trend_feature(xc)
 X.loc[seg_id, 'abs_trend'] = add_trend_feature(xc, abs_values=True)
 X.loc[seg_id, 'abs_mean'] = np.abs(xc ).mean()
X.loc[seg_id, 'abs_std'] = np.abs(xc ).std()
X.loc[seg_id, 'mad'] = xc.mad()
X.loc[seg_id, 'kurt'] = xc.kurtosis()
X.loc[seg_id, 'skew'] = xc.skew()
X.loc[seg_id, 'med'] = xc.median()
X.loc[seg_id, 'Hilbert_mean'] = np.abs(hilbert(xc)).mean()
X.loc[seg_id, 'Hann_window_mean'] =(convolve(xc, hann(150), mode='same')/ sum(hann(150)) ).mean()
X.loc[seg_id, 'classic_sta_lta1_mean'] = classic_sta_lta(xc, 500, 10000 ).mean()
X.loc[seg_id, 'classic_sta_lta2_mean'] = classic_sta_lta(xc, 5000, 100000 ).mean()
X.loc[seg_id, 'classic_sta_lta3_mean'] = classic_sta_lta(xc, 3333, 6666 ).mean()
X.loc[seg_id, 'classic_sta_lta4_mean'] = classic_sta_lta(xc, 10000, 25000 ).mean()
X.loc[seg_id, 'Moving_average_700_mean'] = xc.rolling(window=700 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_1500_mean'] = xc.rolling(window=1500 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_3000_mean'] = xc.rolling(window=3000 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_6000_mean'] = xc.rolling(window=6000 ).mean().mean(skipna=True)
 ewma = pd.Series.ewm
X.loc[seg_id, 'exp_Moving_average_300_mean'] =(ewma(xc, span=300 ).mean() ).mean(skipna=True)
 X.loc[seg_id, 'exp_Moving_average_3000_mean'] = ewma(xc, span=3000 ).mean().mean(skipna=True)
 X.loc[seg_id, 'exp_Moving_average_30000_mean'] = ewma(xc, span=6000 ).mean().mean(skipna=True)
 no_of_std = 2
X.loc[seg_id, 'MA_700MA_std_mean'] = xc.rolling(window=700 ).std().mean()
X.loc[seg_id,'MA_700MA_BB_high_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X.loc[seg_id,'MA_700MA_BB_low_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X.loc[seg_id, 'MA_400MA_std_mean'] = xc.rolling(window=400 ).std().mean()
X.loc[seg_id,'MA_400MA_BB_high_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X.loc[seg_id,'MA_400MA_BB_low_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X.loc[seg_id, 'MA_1000MA_std_mean'] = xc.rolling(window=1000 ).std().mean()
X.loc[seg_id, 'iqr'] = np.subtract(*np.percentile(xc, [75, 25]))
X.loc[seg_id, 'q999'] = np.quantile(xc,0.999)
 X.loc[seg_id, 'q001'] = np.quantile(xc,0.001)
 X.loc[seg_id, 'ave10'] = stats.trim_mean(xc, 0.1)
 for windows in [10, 100, 1000]:
x_roll_std = xc.rolling(windows ).std().dropna().values
x_roll_mean = xc.rolling(windows ).mean().dropna().values
X.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X.loc[seg_id, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X.loc[seg_id, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
 X.loc[seg_id, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)
 X.loc[seg_id, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X.loc[seg_id, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X.loc[seg_id, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X.loc[seg_id, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X.loc[seg_id, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X.loc[seg_id, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X.loc[seg_id, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
X.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 X.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 X.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 X.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 X.loc[seg_id, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X.loc[seg_id, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X.loc[seg_id, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max()<feature_engineering>
df_num_transformed=pd.DataFrame(X_num_transformed.reshape(-1,36),columns=X[num_train_columns].columns )
for seg_id in tqdm_notebook(range(segments)) :
seg = train_df.iloc[seg_id*rows:seg_id*rows+rows]
create_features(seg_id, seg, train_X)
 train_y.loc[seg_id, 'time_to_failure'] = seg['time_to_failure'].values[-1]<normalization>
X_transformed= pd.concat([df_num_transformed,X[cat_columns]],axis=1 )
scaler = StandardScaler()
scaler.fit(train_X)
 scaled_train_X = pd.DataFrame(scaler.transform(train_X), columns=train_X.columns )<load_from_csv>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 test_X = pd.DataFrame(columns=train_X.columns, dtype=np.float64, index=submission.index )<load_from_csv>
warnings.filterwarnings("once" )
for seg_id in tqdm_notebook(test_X.index):
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 create_features(seg_id, seg, test_X )<create_dataframe>
scaled_test_X = pd.DataFrame(scaler.transform(test_X), columns=test_X.columns )<normalization>
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 train_columns = scaled_train_X.columns.values<init_hyperparams>
params = {'num_leaves': 51,
'min_data_in_leaf': 10,
'objective':'regression',
'max_depth': -1,
'learning_rate': 0.001,
"boosting": "gbdt",
"feature_fraction": 0.91,
"bagging_freq": 1,
"bagging_fraction": 0.91,
"bagging_seed": 42,
"metric": 'mae',
"lambda_l1": 0.1,
"verbosity": -1,
"nthread": -1,
"random_state": 42}<train_model>
oof = np.zeros(len(scaled_train_X))
predictions = np.zeros(len(scaled_test_X))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(scaled_train_X,train_y.values)) :
strLog = "fold {}".format(fold_)
 print(strLog)
 X_tr, X_val = scaled_train_X.iloc[trn_idx], scaled_train_X.iloc[val_idx]
y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]
model = lgb.LGBMRegressor(**params, n_estimators = 20000, n_jobs = -1)
 model.fit(X_tr,
y_tr,
eval_set=[(X_tr, y_tr),(X_val, y_val)],
eval_metric='mae',
verbose=1000,
early_stopping_rounds=500)
 oof[val_idx] = model.predict(X_val, num_iteration=model.best_iteration_)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = train_columns
fold_importance_df["importance"] = model.feature_importances_[:len(train_columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += model.predict(scaled_test_X, num_iteration=model.best_iteration_)/ folds.n_splits<save_to_csv>
submission.time_to_failure = predictions
submission.to_csv('submission.csv',index=True )<import_modules>
!pip install fastprogress==0.2.3
!pip install fastai==1.0.61 
<set_options>
!pip install catboost
device = 'cuda' if torch.cuda.is_available() else 'cpu'<define_variables>
cat=CatBoostRegressor(random_state=123,cat_features= cat_columns)
path = Path('/kaggle/input/plant-pathology-2020-fgvc7/' )<load_from_csv>
from skopt.space import Real, Categorical, Integer
from skopt import BayesSearchCV
train_df = pd.read_csv(path/'train.csv')
 test_df = pd.read_csv(path/'test.csv')
 sample_df = pd.read_csv(path/'sample_submission.csv' )<feature_engineering>
search_spaces = {'iterations': Integer(100, 2000),
'depth': Integer(1, 8),
'learning_rate': Real(0.01, 1.0, 'log-uniform'),
'random_strength': Real(1e-9, 10, 'log-uniform'),
'bagging_temperature': Real(0.0, 1.0),
'border_count': Integer(1, 255),
'l2_leaf_reg': Integer(2, 40)}
tuned_cat=BayesSearchCV(cat,search_spaces,cv=3,scoring='neg_mean_squared_error',random_state=123,verbose=1,return_train_score=True,n_jobs=-1)
 tuned_cat.fit(X_transformed,y )
test_df['image_id'] = test_df['image_id'] + '.jpg'
train_df['image_id'] = train_df['image_id'] + '.jpg'<define_variables>
tuned_cat.best_params_
def get_label(row):
if row.healthy == 1:
return 'healthy'
elif row.rust == 1:
return 'rust'
elif row.scab == 1:
return 'scab'
else:
return 'multiple_diseases'<feature_engineering>
embeded_cat_selector = SelectFromModel(tuned_cat.best_estimator_, max_features=X_transformed.shape[1])
 embeded_cat_selector.fit(X_transformed, y )
train_df['label'] = train_df.apply(get_label, axis=1 )<drop_column>
embeded_cat_support = embeded_cat_selector.get_support()
embeded_cat_feature = X_transformed.loc[:,embeded_cat_support].columns.tolist()
print(str(len(embeded_cat_feature)) , 'selected features' )
train_df = train_df[['image_id', 'label']]<count_values>
reduced_features_catboost=embeded_cat_feature
c = Counter(train_df.label), len(train_df)
 c<load_from_csv>
cat_importance_features=pd.DataFrame(zip(tuned_cat.best_estimator_.feature_importances_,X_transformed.columns), columns=['Value','Feature'])
 cat_importance_features[cat_importance_features['Value']>0].shape[0]
psuedo_label_submission = pd.read_csv('/kaggle/input/plant-pathology-fgvc7-2020-submissions/971968_966_964.csv' )<define_variables>
X_catboost_reduced=X_transformed[embeded_cat_feature]
def get_pseudo_label(row):
probs = [row.healthy, row.multiple_diseases, row.rust, row.scab]
label = ['healthy', 'multiple_diseases', 'rust', 'scab'][np.argmax(probs)]
return label<feature_engineering>
cat=CatBoostRegressor(random_state=123,cat_features= catboost_cat_columns)
 search_spaces = {'iterations': Integer(100, 2000),
'depth': Integer(1, 8),
'learning_rate': Real(0.01, 1.0, 'log-uniform'),
'random_strength': Real(1e-9, 10, 'log-uniform'),
'bagging_temperature': Real(0.0, 1.0),
'border_count': Integer(1, 255),
'l2_leaf_reg': Integer(2, 30)}
tuned_cat=BayesSearchCV(cat,search_spaces,cv=3,scoring='neg_mean_squared_error',random_state=123,verbose=1,return_train_score=True,n_jobs=-1)
 tuned_cat.fit(X_catboost_reduced,y )
psuedo_label_submission['label'] = psuedo_label_submission.apply(get_pseudo_label, axis=1 )<feature_engineering>
tuned_cat.best_score_
psuedo_label_submission['image_id'] = psuedo_label_submission['image_id'] + '.jpg'<create_dataframe>
tuned_cat.best_params_
psuedo_label_submission = psuedo_label_submission[['image_id', 'label']]<concatenate>
!pip install lightgbm
train_df_with_pl = pd.concat([train_df, psuedo_label_submission] )<define_variables>
gbm = lgb.LGBMRegressor(random_state=123 )
train_idx_with_pl = train_idx + list(range(len(train_df), len(train_df)+ len(psuedo_label_submission)) )<categorify>
X_transformed_copy=X_transformed.copy()
for col in cat_columns:
X_transformed_copy[col] = X_transformed_copy[col].astype('category')
def get_img_id_from_idx(idx):
return train_df.iloc[idx].image_id<load_pretrained>
param_test ={'num_leaves': Integer(6, 50),
'min_child_samples': Integer(100, 500),
'min_child_weight': [1e-5, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4],
'subsample': Real(0.2,0.8),
'colsample_bytree': Real(0.4, 0.6),
'n_estimators':Integer(100, 2000),
'depth': Integer(1, 8),
'learning_rate': Real(0.01, 1.0, 'log-uniform'),
'bagging_temperature': Real(0.0, 1.0),
'border_count': Integer(1, 255),
'l2_leaf_reg': Integer(2, 30)}
tuned_gbm=BayesSearchCV(gbm,param_test,cv=3,scoring='neg_mean_squared_error',random_state=123,verbose=1,return_train_score=True,n_jobs=-1)
 tuned_gbm.fit(X_transformed_copy,y)
img_size = 1024
train_data = create_databunch_from_img_size(img_size, 32 )<define_variables>
train_sizes, train_scores, test_scores, fit_times, _ = learning_curve(tuned_gbm.best_estimator_, X_transformed_copy, y, cv=3,
return_times=True,random_state=123,
scoring='neg_mean_squared_error')
 train_scores_mean = np.mean(train_scores, axis=1)
 test_scores_mean = np.mean(test_scores, axis=1)
 fig = go.Figure()
fig.add_trace(go.Scatter(x=train_sizes, y=train_scores_mean,
mode='lines+markers',
name='Training score'))
fig.add_trace(go.Scatter(x=train_sizes, y=test_scores_mean,
mode='lines+markers',
name='Cross Validation score'))
fig.show()
train_data.show_batch()<categorify>
embeded_gbm_selector = SelectFromModel(tuned_gbm.best_estimator_, max_features=X_transformed_copy.shape[1])
 embeded_gbm_selector.fit(X_transformed_copy, y )
class ROCAUCScore(Callback):
def on_epoch_begin(self, **kwargs):
self.targets, self.preds = [], []
def on_batch_end(self, last_output, last_target, **kwargs):
self.targets.extend(last_target.tolist())
 self.preds.extend(list(map(scipy.special.softmax, last_output.tolist())))
 def on_epoch_end(self, last_metrics, **kwargs):
sc = roc_auc_score(self.targets, self.preds, multi_class='ovr')
 return add_metrics(last_metrics, sc )<train_model>
embeded_gbm_support = embeded_gbm_selector.get_support()
embeded_gbm_feature = X_transformed_copy.loc[:,embeded_gbm_support].columns.tolist()
print(str(len(embeded_gbm_feature)) , 'selected features' )
class StopTrainingAtEpoch(Callback):
def __init__(self, learn, n):
self.learn = learn
self.n = n
def on_epoch_end(self, epoch, **kwargs):
if epoch == self.n:
self.learn.save('model-' + str(self.n))
return {'stop_training': True}<find_best_params>
X_gbm_reduced=X_transformed_copy[embeded_gbm_feature]
ce_weight = torch.tensor([1.0, 4.0, 1.0, 1.0] ).to(device)
 ce_weight=None
print(ce_weight)
 learn = cnn_learner(train_data, models.resnet18, metrics=[ROCAUCScore() , accuracy, error_rate, Precision(average='macro'), Recall(average='macro'), FBeta(beta=1.0, average='macro')],
loss_func=CrossEntropyFlat(reduction='mean', weight=ce_weight), ps=[0.5, 0.5, 0.5], wd=0.01,
path='/kaggle/working', callback_fns=[ShowGraph])
<load_pretrained>
param_test ={'num_leaves': Integer(6, 50),
'min_child_samples': Integer(100, 500),
'min_child_weight': [1e-5, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4],
'subsample': Real(0.2,0.8),
'colsample_bytree': Real(0.4, 0.6),
'n_estimators':Integer(100, 2000),
'depth': Integer(1, 8),
'learning_rate': Real(0.01, 1.0, 'log-uniform'),
'bagging_temperature': Real(0.0, 1.0),
'border_count': Integer(1, 255),
'l2_leaf_reg': Integer(2, 30)}
tuned_gbm=BayesSearchCV(gbm,param_test,cv=3,scoring='neg_mean_squared_error',random_state=123,verbose=1,return_train_score=True,n_jobs=-1)
 tuned_gbm.fit(X_gbm_reduced,y)
learn.load('/kaggle/input/plant-pathology-2020-fgvc7-best-single-model/stage-05');
learn.unfreeze()<load_pretrained>
tuned_gbm.best_score_
img_size = 1024
train_data = create_databunch_from_img_size(img_size, 16)
 learn.data = train_data<save_model>
train_sizes, train_scores, test_scores, fit_times, _ = learning_curve(tuned_gbm.best_estimator_, X_gbm_reduced, y, cv=3,
return_times=True,random_state=123,
scoring='neg_mean_squared_error')
 train_scores_mean = np.mean(train_scores, axis=1)
 test_scores_mean = np.mean(test_scores, axis=1)
 fig = go.Figure()
fig.add_trace(go.Scatter(x=train_sizes, y=train_scores_mean,
mode='lines+markers',
name='Training score'))
fig.add_trace(go.Scatter(x=train_sizes, y=test_scores_mean,
mode='lines+markers',
name='Cross Validation score'))
fig.show()
learn.save('stage-05' )<train_model>
test_df_copy=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )
learn.fit_one_cycle(20, slice(1e-7, 1e-2 / 10), start_epoch=0, callbacks=[StopTrainingAtEpoch(learn, 19)] )<save_model>
test_df=test_df_copy.copy()
learn.save('stage-06' )<load_pretrained>
cat_columns=test_df.select_dtypes(include=['O','object'] ).columns
for cols in cat_columns:
test_df[cols].fillna(test_df[cols].mode() [0],inplace=True )
learn.load('stage-06');<train_model>
outlier_cols=test_df.skew().index[test_df.skew().values>1]
learn.validate(learn.data.train_dl )<train_model>
num_columns=test_df.select_dtypes(exclude=['O','object'] ).columns
for cols in num_columns:
if cols in outlier_cols:
test_df[cols].fillna(test_df[cols].median() ,inplace=True)
 else:
test_df[cols].fillna(test_df[cols].mean() ,inplace=True )
learn.validate(learn.data.valid_dl )<find_best_params>
test_df.isnull().sum().index[test_df.isnull().sum().values>0]
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()<choose_model_class>
test_df.drop(["Id"],axis=1,inplace=True )
valid_tta_preds, y, losses = learn.TTA(scale=1.10, with_loss=True )<compute_test_metric>
test_num_transformed=qt.transform(test_df[num_train_columns] )
accuracy(valid_tta_preds, y )<compute_test_metric>
df_test_num_transformed=pd.DataFrame(test_num_transformed.reshape(-1,36),columns=test_df[num_train_columns].columns )
confusion_matrix(y.cpu().numpy() , valid_tta_preds.argmax(1 ).cpu().numpy() )<import_modules>
outlier_cols=df_test_num_transformed.skew().index[df_test_num_transformed.skew().values>1]
for cols in outlier_cols:
df_test_num_transformed[cols]=winsorize(df_test_num_transformed[cols], limits=[0.05, 0.05] )
learn.export('model.pkl' )<predict_on_test>
X_test_num_transformed=rs.transform(df_test_num_transformed )
preds, y = learn.get_preds(DatasetType.Test )<save_to_csv>
X_test_num_transformed=pd.DataFrame(X_test_num_transformed.reshape(-1,36),columns=test_df[num_train_columns].columns )
sample_df.iloc[:,1:] = preds.numpy()
sample_df.to_csv('submission.csv', index=False )<install_modules>
test_transformed= pd.concat([X_test_num_transformed,test_df[cat_columns]],axis=1 )
!pip install -q pretrainedmodels<train_model>
X_test_catboost_reduced=test_transformed[embeded_cat_feature]
def model_f(pretrained=True,**kwargs):
return pretrainedmodels.se_resnext50_32x4d(num_classes=1000,pretrained='imagenet' )<load_from_csv>
test_cat_predictions=tuned_cat.predict(X_test_catboost_reduced )
path = Path('/kaggle/input/plant-pathology-2020-fgvc7')
 train = pd.read_csv(path/'train.csv')
 test = pd.read_csv(path/'test.csv')
 train['label'] = train.set_index('image_id' ).idxmax(axis=1 ).reset_index() [0]
train<define_variables>
test_transformed_copy=test_transformed.copy()
for col in cat_columns:
test_transformed_copy[col] = test_transformed_copy[col].astype('category')
 X_test_gbm_reduced=test_transformed_copy[embeded_gbm_feature]
test_gbm_predictions=tuned_gbm.predict(X_test_gbm_reduced)
pseudos = ['Test_1149', 'Test_1158','Test_1321','Test_1590','Test_1634','Test_168','Test_1728','Test_1801','Test_211','Test_282','Test_506','Test_543','Test_714','Test_727','Test_921','Test_932','Test_987','Test_991'] 
<create_dataframe>
pd.DataFrame(np.expm1(test_gbm_predictions))
c=['image_id', 'healthy', 'multiple_diseases', 'rust', 'scab', 'label']
pseudos = pd.DataFrame([(x, 0,1,0,0,'multiple_diseases')for x in pseudos], columns=c)
 pseudos<concatenate>
final_predictions=(0.7*np.expm1(test_cat_predictions)) +(0.3*np.expm1(test_gbm_predictions))
train = pd.concat([train, pseudos], axis=0 )<define_variables>
sub = pd.DataFrame()
sub['Id'] = test_df_copy['Id']
sub['SalePrice'] = final_predictions
sub.head()
dups = [["Test_1800", "Train_558"], ["Train_616", "Train_966"], ["Test_383", "Test_635"], ["Train_500", "Train_647"], ["Test_1691", "Test_683"], ["Test_431", "Test_776"], ["Test_218", "Train_1598"], ["Test_611", "Train_1698"], ["Train_909", "Train_914"], ["Test_1688", "Train_1154"], ["Test_700", "Train_1142"], ["Test_1047", "Train_1733"], ["Train_592", "Train_782"], ["Train_1675", "Train_417"], ["Test_1450", "Train_761"], ["Test_989", "Train_1347"], ["Test_1117", "Train_105"], ["Test_119", "Train_772"], ["Test_1503", "Test_232"], ["Test_1264", "Test_708"], ["Test_483", "Train_1407"], ["Test_1212", "Test_570"], ["Train_174", "Train_1817"], ["Test_1722", "Test_889"], ["Test_693", "Train_1277"], ["Test_1494", "Train_937"], ["Test_938", "Train_861"], ["Test_53", "Train_114"], ["Test_581", "Train_1615"], ["Train_1731", "Train_591"], ["Test_1802", "Train_574"], ["Train_665", "Train_694"], ["Train_1634", "Train_921"], ["Train_1115", "Train_1683"], ["Train_1661", "Train_815"], ["Test_1062", "Train_1368"], ["Test_1720", "Test_459"], ["Test_1561", "Test_443"], ["Train_759", "Train_828"], ["Test_1671", "Test_899"], ["Train_1581", "Train_1613"], ["Test_768", "Train_851"], ["Test_1203", "Test_1600"], ["Test_797", "Train_1585"], ["Test_1163", "Test_542"], ["Test_886", "Train_1541"], ["Test_1011", "Train_608"], ["Train_1173", "Train_379"], ["Test_1588", "Test_376"], ["Test_1675", "Train_1066"], ["Test_1020", "Train_683"], ["Test_150", "Test_1501"], ["Train_1678", "Train_931"], ["Test_965", "Train_1341"], ["Train_1587", "Train_1593"], ["Train_341", "Train_763"], ["Test_895", "Test_979"], ["Train_162", "Train_356"], ["Test_1407", "Train_1703"], ["Test_1322", "Train_1600"], ["Test_1524", "Train_919"], ["Train_1150", "Train_1365"], ["Train_1392", "Train_438"], ["Test_829", "Train_1505"], ["Test_826", "Train_847"], ["Train_115", "Train_1709"]]
dups = pd.DataFrame(dups, columns=['col1', 'col2'])
 dups.head()
dups['col1t'] = dups['col1'].str.split('_' ).str.get(0)
 dups['col2t'] = dups['col2'].str.split('_' ).str.get(0)
 dups.head()<merge>
sub.to_csv('submission.csv',index=False )