Kaggle competition: CareerCon 2019 - Help Navigate Robots
<SOS> metric: categorizationaccuracy Kaggle data source: career-con-2019<concatenate>
warnings.filterwarnings('ignore')
 logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.ERROR)
 pd.set_option("max_columns", 200)
 pd.set_option("max_rows", 200)
 gc.enable()
def df_parallelize_run(func, t_split):
num_cores = np.min([__n_cores, len(t_split)])
 pool = Pool(num_cores)
 df = pd.concat(pool.map(func, t_split), axis=0)
 pool.close()
pool.join()
return df<create_dataframe>
x_train = pd.read_csv('.. /input/X_train.csv')
 y_train = pd.read_csv('.. /input/y_train.csv')
 x_test = pd.read_csv('.. /input/X_test.csv' )
def add_stats_on_the_row(df_stats, df_result = None):
input_list = list(itertools.product(*[list(df_stats.columns), list(df_stats.index.values)]))
columns = [f'{e[0]}_{str(e[1])}' for e in input_list]
res = pd.DataFrame(df_stats.values.reshape(1, df_stats.shape[0] * df_stats.shape[1], order="F"), columns = columns)
 if df_result is None:
return res
else:
return pd.concat([df_result, res], axis = 1)
 print("Little example, how I put statistics on a unique row :")
 df_ex = pd.DataFrame({'a':[100, 1, 2], 'b':[10, 20, 3], 'c':[20, 60, 7]})
 df_stats = add_stats_on_the_row(df_ex[["a", "c"]].agg(["min", "mean", "max"]))
print("Let's create a litte data frame
{}
And use this function to compute statistics :
{}".
 format(df_ex, df_stats))
del df_ex, df_stats<categorify>
tsfresh_train = extract_features(x_train.drop('row_id', axis=1), column_id='series_id',
column_sort='measurement_number')
 impute(tsfresh_train);
def lfeat_with_suff(suffix=""):
return [f"sensor_{i+1}{suffix}" for i in range(10)]
lfeat = lfeat_with_suff()
lfeat_abs = lfeat_with_suff("_abs")
 lfeat_fft_real = lfeat_with_suff("_fft_real")
 lfeat_fft_imag = lfeat_with_suff("_fft_imag")
 input_list = [range(10), ["min", "max"]]
list_result = list(itertools.product(*input_list))<feature_engineering>
relevant_train_features = set()
for label in y_train['surface'].unique() :
y_train_binary =(y_train['surface'].values == label ).astype(int)
 print('=='*20); print(y_train_binary); print('=='*20);
X_train_filtered = select_features(tsfresh_train, y_train_binary, fdr_level=0.382)
 print('=='*20);
print("Number of relevant features for class {}: {}/{}".format(
 label, X_train_filtered.shape[1], tsfresh_train.shape[1]))
print('=='*20);
relevant_train_features = relevant_train_features.union(set(X_train_filtered.columns))
fs = 100
N = 60001
n = 256
max_f = 20
delta_f = fs / n
delta_t = n / fs / 2
def STFT_Features(segment_df, segment_id):
segment = [segment_id]
for sensor in segment_df.columns:
x = segment_df[sensor][:N]
if x.isna().sum() > 1000:
segment +=([np.NaN] * 10)
 continue
f, t, Z = scipy.signal.stft(x.fillna(0), fs = fs, window = 'hann', nperseg = n)
 f = f[:round(max_f/delta_f)+1]
Z = np.abs(Z[:round(max_f/delta_f)+1] ).T
th = Z.mean() * 1
Z_pow = Z.copy()
Z_pow[Z < th] = 0
Z_num = Z_pow.copy()
Z_num[Z >= th] = 1
Z_pow_sum = Z_pow.sum(axis = 0)
 Z_num_sum = Z_num.sum(axis = 0)
 A_pow = Z_pow_sum[round(10/delta_f):].sum()
A_num = Z_num_sum[round(10/delta_f):].sum()
BH_pow = Z_pow_sum[round(5/delta_f):round(8/delta_f)].sum()
BH_num = Z_num_sum[round(5/delta_f):round(8/delta_f)].sum()
BL_pow = Z_pow_sum[round(1.5/delta_f):round(2.5/delta_f)].sum()
BL_num = Z_num_sum[round(1.5/delta_f):round(2.5/delta_f)].sum()
C_pow = Z_pow_sum[round(0.6/delta_f):round(1.2/delta_f)].sum()
C_num = Z_num_sum[round(0.6/delta_f):round(1.2/delta_f)].sum()
D_pow = Z_pow_sum[round(2/delta_f):round(4/delta_f)].sum()
D_num = Z_num_sum[round(2/delta_f):round(4/delta_f)].sum()
segment += [A_pow, A_num, BH_pow, BH_num, BL_pow, BL_num, C_pow, C_num, D_pow, D_num]
features = [f"sensor_{i+1}_{f}" for i in range(10)for f in ["A_pow", "A_num", "BH_pow", "BH_num", "BL_pow", "BL_num"
, "C_pow", "C_num", "D_pow", "D_num"]]
df = pd.DataFrame(np.array([segment]), columns = ["segment_id"] + features)
 df["segment_id"] = df["segment_id"].astype(np.int32)
 return df<load_from_csv>
tsfresh_test = extract_features(x_test.drop('row_id', axis=1), column_id='series_id',
column_sort='measurement_number')
 impute(tsfresh_test);
def features_engineering_one_file(onefile, source):
sequence = pd.read_csv(f'/kaggle/input/predict-volcanic-eruptions-ingv-oe/{source}/{onefile}.csv')
 res = STFT_Features(sequence[lfeat], onefile)
 res = pd.concat([res, pd.DataFrame(sequence[lfeat].isna().sum().values.reshape(1, 10, order="F")
 , columns=[f"sensor_{i+1}_nan" for i in range(10)])], axis=1)
 sequence.fillna(0, inplace = True)
 sequence[lfeat_abs] = sequence[lfeat].abs()
ft = np.fft.fft(sequence[lfeat], axis=0)
 sequence[lfeat_fft_real] = np.real(ft)
 sequence[lfeat_fft_imag] = np.imag(ft)
 lfun = ["mean", "std", "min", "max", "mad", "skew", "kurtosis"]
lquantiles = [.01,.05,.1,.25,.5,.75,.9,.95,.99]
res = add_stats_on_the_row(sequence[lfeat + lfeat_abs + lfeat_fft_real + lfeat_fft_imag].agg(lfun), res)
 res = add_stats_on_the_row(sequence[lfeat + lfeat_abs + lfeat_fft_real + lfeat_fft_imag].quantile(lquantiles), res)
 return res<create_dataframe>
tsfresh_train = tsfresh_train[list(relevant_train_features)]
tsfresh_test = tsfresh_test[list(relevant_train_features)]
%%time
def create_data_for_model(df, source="train"):
func_partial_fe = partial(features_engineering_one_file, source = source)
 df_set = list()
df_set.append(df_parallelize_run(func_partial_fe, df.index[:__dataframe_size]))
df_set = pd.concat(df_set)
 df_set.reset_index(inplace = True, drop=True)
 df_set.set_index("segment_id", inplace=True)
 return df_set
if __create_dataframes:
print("Train dataset creation...")
 train_set = create_data_for_model(df = y_train)
 if __dataframe_size > 100:
print("Test dataset creation...")
 test_set = create_data_for_model(df = y_pred, source = "test")
 else:
train_set = pickle.load(open(__path_to_my_data + "ingv_train_set.pkl", "rb"))
test_set = pickle.load(open(__path_to_my_data + "ingv_test_set.pkl", "rb"))
pickle.dump(train_set, open("ingv_train_set.pkl", "wb"))
if __dataframe_size > 100:
pickle.dump(test_set, open("ingv_test_set.pkl", "wb"))<count_missing_values>
fac_surfaces, surfaces = y_train['surface'].factorize()
train_data = lgb.Dataset(tsfresh_train, label=fac_surfaces)
 params={'learning_rate': 0.1, 'objective':'multiclass', 'metric':'multi_error',
'num_class':9, 'verbose': 1, 'random_state':311,
'bagging_fraction': 0.7, 'feature_fraction': 1.0}
num_round = 15000
light = lgb.train(params, train_data, num_round)
 pred = light.predict(tsfresh_test)
 feature_importances = light.feature_importance()
feature_names = tsfresh_test.columns.values
a=train_set.sum(axis=0, skipna=False)
 print("Features with NAN values :
{}".format(list(a[a.isna() ].index)) )<categorify>
pred[:,0].shape
fac_surfaces, surfaces = y_train['surface'].factorize()
final_pred = pd.Series(np.argmax(pred, axis=1))
surface_dict = {}
for n, s in enumerate(surfaces):
surface_dict[n] = s
final_pred = final_pred.map(surface_dict )
<count_unique_values><EOS> 
submission = pd.DataFrame({
"series_id": list(range(3816)) ,
"surface": final_pred
})
 submission.to_csv('submission.csv', index=False )