Kaggle competition: CareerCon 2019 - Help Navigate Robots
<SOS> metric: categorizationaccuracy Kaggle data source: career-con-2019<train_model>
%load_ext autoreload
%autoreload 2
%matplotlib inline
def fit_and_predict_with_lgbm(df_train, params
, seed=__seed, features=None, X_test=None, check_feat_importance = True, n_folds=__n_folds
, early_stopping_rounds = 100):
folds = KFold(n_splits = n_folds, shuffle = True, random_state = seed)
 list_mae, list_r2 = [], []
predictions = None
feature_importance_df = pd.DataFrame()
feat_imp={}
params["random_state"] = seed + 1
params["bagging_seed"] = seed - 1
if not features is None:
lfeat = features
else:
lfeat = [f for f in df_train.columns if f not in ["time_to_eruption"]]
print("Train DF shape : {} rows and {} features".format(df_train.shape[0], len(lfeat)))
 if not X_test is None:
X_test = X_test[lfeat]
print("Test DF shape : {} rows and {} features".format(X_test.shape[0], X_test.shape[1]))
predictions = np.zeros(len(X_test))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(df_train)) :
print("Fold n°{}".format(fold_+1))
X_tr, X_val = df_train[lfeat].iloc[trn_idx], df_train[lfeat].iloc[val_idx]
y_tr, y_val = df_train['time_to_eruption'].iloc[trn_idx], df_train['time_to_eruption'].iloc[val_idx]
model = lgbm.LGBMRegressor(**params, n_estimators = 3000, n_jobs = -1)
 model.fit(X_tr, y_tr,
eval_set=[(X_tr, y_tr),(X_val, y_val)], eval_metric='mae',
verbose=1000, early_stopping_rounds = early_stopping_rounds)
 y_val_pred = model.predict(X_val, num_iteration=model.best_iteration_)
 _mae = mean_absolute_error(y_val_pred, y_val); list_mae.append(_mae)
 _r2 = r2_score(y_val_pred, y_val); list_r2.append(_r2)
 print("Scores on valid set for fold {} : MAE {:,.0f} & R2 {:.3f}".format(fold_+1, _mae, _r2))
if not X_test is None:
predictions += model.predict(X_test[lfeat], num_iteration=model.best_iteration_)/ folds.n_splits
if check_feat_importance:
fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = X_tr.columns
fold_importance_df["importance"] = model.feature_importances_[:len(X_tr.columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 for feat in X_tr.columns:
if fold_ == 0:
feat_imp[feat] = []
temp_df = X_val.copy()
temp_df[feat] = np.random.permutation(temp_df[feat])
 y_temp = model.predict(temp_df, num_iteration=model.best_iteration_)
 feat_imp[feat].append(mean_absolute_error(y_temp, y_val)- _mae)
 _mae = np.array(list_mae ).mean()
_r2 = np.array(list_r2 ).mean()
print("Mean scores on valid sets : MAE {:,.0f} & R2 {:.3f}".format(_mae, _r2))
feat_imp_df = None
if check_feat_importance:
feat_imp_df = pd.DataFrame(feat_imp ).transpose()
lfeat = [i for i in range(__n_folds)]
lfeat_sign = [f"sign_{i}" for i in range(n_folds)]
feat_imp_df[lfeat_sign] = 0
for mae_loss, mae_loss_sign in zip(lfeat, lfeat_sign):
feat_imp_df.loc[feat_imp_df[mae_loss]>0, mae_loss_sign] = 1
feat_imp_df["sum_sign"] = feat_imp_df[lfeat_sign].sum(axis=1)
 return {"pred":predictions, "feat_imp":feat_imp_df, "list_mae":list_mae, "mean_mae_valid":_mae
, "list_r2":list_r2, "mean_r2_valid":_r2, "plot_feat_imp":feature_importance_df}
params = {
'num_leaves': 28,
'min_data_in_leaf': 10,
'learning_rate': 0.15,
'max_bins': 50,
"feature_fraction": 0.5,
"bagging_freq": 1,
"bagging_fraction": 0.8,
"lambda_l1": 0.1,
"boosting": "gbdt",
'objective':'regression',
"metric": 'mae',
"verbosity": -1,
"nthread": -1,
}<predict_on_test>
from fastai.imports import *
from pandas_summary import DataFrameSummary
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from IPython.display import display
from sklearn import metrics
from fastai.tabular import *
from fastai.collab import *
from IPython.display import HTML
from sklearn.preprocessing import LabelEncoder
%%time
result1 = fit_and_predict_with_lgbm(train_set, params)
 feat_imp_df = result1["feat_imp"]
feats = list(feat_imp_df.loc[feat_imp_df["sum_sign"] == __n_folds].index)
 print("
There are {} features with no loss of MAE on the {} folds.
 ".format(len(feats), __n_folds))<compute_train_metric>
df_raw = pd.read_csv(f'{PATH}/X_train.csv', low_memory=False)
 y_raw = pd.read_csv(f'{PATH}/y_train.csv', low_memory=False)
 df_test = pd.read_csv(f'{PATH}/X_test.csv', low_memory=False)
 sub = pd.read_csv(f'{PATH}/sample_submission.csv' )
%%time
result2 = fit_and_predict_with_lgbm(train_set, params, seed=__seed+10, features=feats, X_test = test_set)
 print("
Results summary :")
 print("CV n°1 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result1["mean_mae_valid"], result1["mean_r2_valid"]))
print("CV n°2 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result2["mean_mae_valid"], result2["mean_r2_valid"]))
feat_imp_df = result2["feat_imp"]
feats = list(feat_imp_df.loc[feat_imp_df["sum_sign"] == __n_folds].index)
 print("
There are {} features with no loss of MAE on the {} folds.
 ".format(len(feats), __n_folds))
make_file_submission(result2["pred"], "submission2.csv" )<compute_train_metric>
df_raw[df_raw.isnull().any(axis=1)]
%%time
result3 = fit_and_predict_with_lgbm(train_set, params, seed=__seed+69, features=feats, X_test = test_set)
 print("
Results summary :")
 print("CV n°1 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result1["mean_mae_valid"], result1["mean_r2_valid"]))
print("CV n°2 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result2["mean_mae_valid"], result2["mean_r2_valid"]))
print("CV n°3 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result3["mean_mae_valid"], result3["mean_r2_valid"]))
feat_imp_df = result3["feat_imp"]
feats = list(feat_imp_df.loc[feat_imp_df["sum_sign"] == __n_folds].index)
 print("
There are {} features with no loss of MAE on the {} folds.
 ".format(len(feats), __n_folds))
make_file_submission(result3["pred"], "submission3.csv" )<compute_train_metric>
y_raw[y_raw.isnull().any(axis=1)]
%%time
result4 = fit_and_predict_with_lgbm(train_set, params, seed=__seed-41, features=feats, X_test = test_set)
 print("
Results summary :")
 print("CV n°1 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result1["mean_mae_valid"], result1["mean_r2_valid"]))
print("CV n°2 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result2["mean_mae_valid"], result2["mean_r2_valid"]))
print("CV n°3 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result3["mean_mae_valid"], result3["mean_r2_valid"]))
print("CV n°4 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result4["mean_mae_valid"], result4["mean_r2_valid"]))
feat_imp_df = result4["feat_imp"]
feats = list(feat_imp_df.loc[feat_imp_df["sum_sign"] == __n_folds].index)
 print("
There are {} features with no loss of MAE on the {} folds.
 ".format(len(feats), __n_folds))
make_file_submission(result4["pred"], "submission4.csv" )<compute_train_metric>
df_test[df_test.isnull().any(axis=1)]
%%time
result5 = fit_and_predict_with_lgbm(train_set, params, seed=__seed-41, features=feats, X_test = test_set)
 print("
Results summary :")
 print("CV n°1 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result1["mean_mae_valid"], result1["mean_r2_valid"]))
print("CV n°2 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result2["mean_mae_valid"], result2["mean_r2_valid"]))
print("CV n°3 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result3["mean_mae_valid"], result3["mean_r2_valid"]))
print("CV n°4 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result4["mean_mae_valid"], result4["mean_r2_valid"]))
print("CV n°4 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result5["mean_mae_valid"], result5["mean_r2_valid"]))
feat_imp_df = result5["feat_imp"]
feats = list(feat_imp_df.loc[feat_imp_df["sum_sign"] == __n_folds].index)
 print("
There are {} features with no loss of MAE on the {} folds.
 ".format(len(feats), __n_folds))
make_file_submission(result5["pred"], "submission5.csv" )<filter>
le = LabelEncoder()
le.fit(y_raw['surface'])
 y_raw['surface'] = le.transform(y_raw['surface'] )
feat_imp_df[feat_imp_df["sum_sign"] != __n_folds]<split>
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z
def fe(df):
df['total_angular_velocity'] =(df['angular_velocity_X'] ** 2 + df['angular_velocity_Y'] ** 2 + df['angular_velocity_Z'] ** 2)** 0.5
df['total_linear_acceleration'] =(df['linear_acceleration_X'] ** 2 + df['linear_acceleration_Y'] ** 2 + df['linear_acceleration_Z'] ** 2)** 0.5
df['total_xyz'] =(df['orientation_X']**2 + df['orientation_Y']**2 +df['orientation_Z'])**0.5
df['acc_vs_vel'] = df['total_linear_acceleration'] / df['total_angular_velocity']
temp_df = pd.DataFrame()
for col in df.columns[3:]:
temp_df[col + '_mean'] = df.groupby(['series_id'])[col].mean()
temp_df[col + '_median'] = df.groupby(['series_id'])[col].median()
temp_df[col + '_max'] = df.groupby(['series_id'])[col].max()
temp_df[col + '_min'] = df.groupby(['series_id'])[col].min()
temp_df[col + '_std'] = df.groupby(['series_id'])[col].std()
temp_df[col + '_range'] = temp_df[col + '_max'] - temp_df[col + '_min']
temp_df[col + '_maxtoMin'] = temp_df[col + '_max'] / temp_df[col + '_min']
temp_df[col + '_mean_abs_chg'] = df.groupby(['series_id'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
temp_df[col + '_abs_min'] = df.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 temp_df[col + '_abs_max'] = df.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 temp_df[col + '_abs_avg'] =(temp_df[col + '_abs_min'] + temp_df[col + '_abs_max'])/2
return temp_df
df = pd.concat([train_set.drop("time_to_eruption", axis=1), test_set], axis=0)
 ts = train_set.shape
li = df.iloc[:ts[0]].index.values
df["is_test"] = 1
df.loc[li, "is_test"] = 0
print(df["is_test"].value_counts() , "
")
 feat=[f"sensor_{i+1}_d9_d1" for i in range(10)]
tr, te, tr_y, te_y = train_test_split(df[feat], df["is_test"], test_size=0.33, random_state=__seed, shuffle=True)
 param = {'num_leaves': 30,
'min_data_in_leaf': 30,
'max_depth': -1,
'learning_rate': 0.1,
"min_child_samples": 20,
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9 ,
"bagging_seed": __seed,
"boosting": "gbdt",
'objective':'binary',
"metric": 'auc',
"verbosity": -1}
clf = lgbm.LGBMClassifier(**param, n_estimators = 1000, n_jobs = -1)
 clf.fit(tr, tr_y, eval_set = [(tr, tr_y),(te, te_y)], verbose=50, early_stopping_rounds = 100)
 feature_importances = pd.DataFrame(clf.feature_importances_
, index = tr.columns, columns=['importance'] ).sort_values('importance', ascending=False)
 feature_importances = feature_importances.reset_index()
feature_importances.columns = ['feature', 'importance']
fig, ax = plt.subplots(figsize =(18, 8))
sns.set()
plt.subplot(1, 1, 1);
sns.barplot(x="importance", y="feature", orient='h', data=feature_importances.head(50)) ;
plt.title('Feature Importance to detect Train or Test obs');<set_options>
%%time
df_train = fe(df_raw)
 df_test = fe(df_test )
pd.options.display.max_columns = None
%matplotlib inline 
<define_variables>
df_train['surface'] = y_raw['surface']
KAGGLE = True
if KAGGLE:
DIR = '.. /input/predict-volcanic-eruptions-ingv-oe'
else:
DIR = './predict-volcanic-eruptions-ingv-oe/'<load_from_csv>
def create_download_link(title = "Download CSV file", filename = "data.csv"):
html = '<a href={filename}>{title}</a>'
html = html.format(title=title,filename=filename)
 return HTML(html )
train = pd.read_csv(os.path.join(DIR, 'train.csv'))
test = pd.read_csv(os.path.join(DIR, 'sample_submission.csv'))
train<feature_engineering>
train['h:m:s'] =(train['time_to_eruption'] 
.apply(lambda x:datetime.timedelta(seconds = x/100)))
 train<sort_values>
df, y = df_train.drop('surface',axis=1), df_train.surface
sample_df =(train.sort_values('time_to_eruption')
.reset_index() 
.rename(columns={'index': 'train_id'}))
sample_df = sample_df[sample_df.index %(len(train)// 5)== 5].reset_index(drop = True)
 sample_ids = sample_df['segment_id'].values
sample_df<define_variables>
fs = 100
N = len(segment_df)
 n = 256
max_f = 20
delta_f = fs / n
delta_t = n / fs / 2<load_from_csv>
def print_score(m):
res = [ m.score(X_train, y_train), m.score(X_valid, y_valid)]
if hasattr(m, 'oob_score_'): res.append(m.oob_score_)
 print(res )
def make_features(tgt):
tgt_df = train if tgt == 'train' else test
feature_set = []
for segment_id in tqdm(tgt_df['segment_id']):
segment_df = pd.read_csv(os.path.join(DIR,f'{tgt}/{segment_id}.csv'))
segment = [segment_id]
for sensor in segment_df.columns:
x = segment_df[sensor][:N]
if x.isna().sum() > 1000:
segment +=([np.NaN] * 10)
 continue
f, t, Z = scipy.signal.stft(x.fillna(0), fs = fs, window = 'hann', nperseg = n)
 f = f[:round(max_f/delta_f)+1]
Z = np.abs(Z[:round(max_f/delta_f)+1] ).T
th = Z.mean() * 1
Z_pow = Z.copy()
Z_pow[Z < th] = 0
Z_num = Z_pow.copy()
Z_num[Z >= th] = 1
Z_pow_sum = Z_pow.sum(axis = 0)
 Z_num_sum = Z_num.sum(axis = 0)
 A_pow = Z_pow_sum[round(10/delta_f):].sum()
A_num = Z_num_sum[round(10/delta_f):].sum()
BH_pow = Z_pow_sum[round(5/delta_f):round(8/delta_f)].sum()
BH_num = Z_num_sum[round(5/delta_f):round(8/delta_f)].sum()
BL_pow = Z_pow_sum[round(1.5/delta_f):round(2.5/delta_f)].sum()
BL_num = Z_num_sum[round(1.5/delta_f):round(2.5/delta_f)].sum()
C_pow = Z_pow_sum[round(0.6/delta_f):round(1.2/delta_f)].sum()
C_num = Z_num_sum[round(0.6/delta_f):round(1.2/delta_f)].sum()
D_pow = Z_pow_sum[round(2/delta_f):round(4/delta_f)].sum()
D_num = Z_num_sum[round(2/delta_f):round(4/delta_f)].sum()
segment += [A_pow, A_num, BH_pow, BH_num, BL_pow, BL_num, C_pow, C_num, D_pow, D_num]
feature_set.append(segment)
 cols = ['segment_id']
for i in range(10):
for j in ['A_pow', 'A_num','BH_pow', 'BH_num','BL_pow', 'BL_num','C_pow', 'C_num','D_pow', 'D_num']:
cols += [f's{i+1}_{j}']
feature_df = pd.DataFrame(feature_set, columns = cols)
 feature_df['segment_id'] = feature_df['segment_id'].astype('int')
 return feature_df<merge>
feature_df = make_features('train')
 train_set = pd.merge(train, feature_df, on = 'segment_id')
 train_set<split>
dep_var = 'surface'
cont_vars = list(df_test.columns)
 procs = [FillMissing]
df = train_set.drop(['segment_id', 'time_to_eruption','h:m:s'], axis=1)
 y = train_set['time_to_eruption']
X_train, X_val, y_train, y_val = train_test_split(df, y,
random_state = 42,
test_size = 0.2,
shuffle = True)
 features = X_train.columns.tolist()
cat_features = {}<train_model>
data =(TabularList.from_df(df_train, path=path, cont_names=cont_vars, procs=procs,)
.split_by_idx(list(range(3801,3810)))
.label_from_df(cols=dep_var)
.add_test(TabularList.from_df(df_test, path=path, cont_names=cont_vars)) 
.databunch() )
def do_lgb(X_train, y_train, X_val, y_val):
params = {'objective': 'rmse',
'metric': 'rmse',
'max_depth':14,
'min_data_in_leaf':5,
'num_leaves': 2**7 - 1,
'learning_rate': 0.05,
'feature_fraction': 0.7,
'bagging_fraction': 0.5,
'bagging_freq': 5,
'lambda_l1':80,
'num_iterations': 10000,
'seed': 42,
'verbose': 1
}
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train)
 evals_result = {}
model = lgb.train(
 params,
lgb_train,
valid_sets =(lgb_train, lgb_eval),
feature_name = features,
categorical_feature = cat_features,
verbose_eval = 100,
evals_result = evals_result,
early_stopping_rounds = 200)
 return model<compute_test_metric>
data.show_batch(rows=5)
lgb_model = do_lgb(X_train, y_train, X_val, y_val )<merge>
learn = tabular_learner(data, layers=[1000,500], ps=[0.005,0.05], emb_drop=0.08,
metrics=accuracy )
feature_df = make_features('test')
 test_set = pd.merge(test, feature_df, on = 'segment_id')
 test_set<predict_on_test>
learn.model
preds = lgb_model.predict(test_set.drop(['segment_id', 'time_to_eruption'], axis=1))
test['time_to_eruption'] = preds
test[['segment_id','time_to_eruption']]<save_to_csv>
learn.fit_one_cycle(5, 2e-3, wd=0.06 )
test[['segment_id','time_to_eruption']].to_csv('submission.csv', index=False )<load_from_csv>
learn.fit_one_cycle(6, 2e-3,wd=0.02 )
train = pd.read_csv('.. /input/ingv-tsfresh-7730/train.csv', sep = ';')
 train.set_index('Unnamed: 0', inplace = True)
 test = pd.read_csv('.. /input/ingv-tsfresh-7730/test.csv', sep = ';')
 test.set_index('Unnamed: 0', inplace = True )<feature_engineering>
learn.fit_one_cycle(10, 1e-7,wd=0.02 )
<split>
test_preds = learn.get_preds(DatasetType.Test)[1]
submit1 = pd.concat([test_seriesid,pd.Series(test_preds)],axis=1)
 submit1.columns =['series_id','surface']
%%time
Y = train['time_to_eruption']
X = train.drop(['time_to_eruption'], axis = 1)
 X_test = test
n_fold = 5
cv = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 oof = np.zeros(len(X))
cat_prediction = np.zeros(len(X_test))
mae, r2 = [], []
PARAMS = {
'random_seed': 42,
'eval_metric': 'MAE',
'task_type': 'GPU',
}
for fold_n,(train_index, valid_index)in enumerate(cv.split(X)) :
print('
Fold', fold_n, 'started at', ctime())
 X_train = X.iloc[train_index,:]
X_valid = X.iloc[valid_index,:]
Y_train = Y.iloc[train_index]
Y_valid = Y.iloc[valid_index]
best_model = CatBoostRegressor(**PARAMS, thread_count = -1)
 train_dataset = Pool(data=X_train,
label=Y_train, 
)
 eval_dataset = Pool(data=X_valid,
label=Y_valid, 
)
 best_model.fit(train_dataset,
use_best_model=True,
verbose = False,
plot = True,
eval_set=eval_dataset,
early_stopping_rounds=100)
 y_pred = best_model.predict(Pool(data=X_valid))
mae.append(mean_absolute_error(Y_valid, y_pred))
r2.append(r2_score(Y_valid, y_pred))
print('MAE: ', mean_absolute_error(Y_valid, y_pred))
print('R2: ', r2_score(Y_valid, y_pred))
cat_prediction += best_model.predict(Pool(data=X_test))
cat_prediction /= n_fold
print('='*45)
 print('CV mean MAE: {0:.4f}, std: {1:.4f}.'.format(np.mean(mae), np.std(mae)))
 print('CV mean R2: {0:.4f}, std: {1:.4f}.'.format(np.mean(r2), np.std(r2)) )<save_to_csv>
learn.get_preds(DatasetType.Test)[0]
submission = pd.DataFrame()
submission['segment_id'] = test.index
submission['time_to_eruption'] = cat_prediction
submission.to_csv('submission.csv', header=True, index=False )<load_pretrained>
preds, _ = learn.get_preds(ds_type=DatasetType.Test)
 labelled_preds = le.inverse_transform(np.array([np.argmax(preds[i])for i in range(len(preds)) ]))
sub['surface'] = labelled_preds
<load_from_csv><EOS> 
sub.to_csv('submission.csv',index=False)
 create_download_link(filename='submission.csv' )