Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<feature_engineering>
warnings.filterwarnings("ignore")
 BLUE = '
RED = '
GREEN = '
train_df['features'] = train_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
test_df['features'] = test_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
print(train_df["features"].head())
 tfidf = CountVectorizer(stop_words='english', max_features=200)
 tr_sparse = tfidf.fit_transform(train_df["features"])
 te_sparse = tfidf.transform(test_df["features"] )<prepare_x_and_y>
Image(".. /input/feature-generation-frame/feature_generation_framework.png" )
train_X = sparse.hstack([train_df[features_to_use], tr_sparse] ).tocsr()
test_X = sparse.hstack([test_df[features_to_use], te_sparse] ).tocsr()
target_num_map = {'high':0, 'medium':1, 'low':2}
train_y = np.array(train_df['interest_level'].apply(lambda x: target_num_map[x]))
print(train_X.shape, test_X.shape )<compute_train_metric>
train = pd.read_csv(".. /input/processed/train_data.csv")
 test = pd.read_csv(".. /input/processed/test_data.csv" )
cv_scores = []
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2016)
 for dev_index, val_index in kf.split(range(train_X.shape[0])) :
dev_X, val_X = train_X[dev_index,:], train_X[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
preds, model = runXGB(dev_X, dev_y, val_X, val_y)
 cv_scores.append(log_loss(val_y, preds))
print(cv_scores)
 break<create_dataframe>
train['fold'] = np.arange(len(train)) // 10
train.loc[(np.arange(len(train)) % 10 > 4), 'fold'] = -1
train = train[train.fold != -1].reset_index().drop('index', axis=1)
 N = train.fold.max()
K = 10
k = N // K
np.random.seed(0)
 permutation = np.random.choice(N, N, replace=False)
 folds = []
for i in range(K):
fold = permutation[i*k:(i+1)*k]
folds.append(fold)
 tmp = train[~np.isin(train.fold, fold)]
preds, model = runXGB(train_X, train_y, test_X, num_rounds=400)
 out_df = pd.DataFrame(preds )<rename_columns>
def build_linear_model(feature):
train['prediction'] = 0
for i in range(K):
val_fold = folds[i]
cv_train = train[(~np.isin(train.fold, val_fold)) ]
cv_val = train[np.isin(train.fold, val_fold)]
feature_train = cv_train[feature].values
target_train = cv_train[TARGET].values
feature_val = cv_val[feature].values
target_val = cv_val[TARGET].values
model = LinearRegression()
model.fit(feature_train.reshape(-1,1), target_train)
 val_predictions = model.predict(feature_val.reshape(-1,1))
train.loc[np.isin(train.fold, val_fold), 'prediction'] = val_predictions
cv_train = train.copy()
feature_train = cv_train[feature].values
target_train = cv_train[TARGET].values
model = LinearRegression()
model.fit(feature_train.reshape(-1,1), target_train)
 score = mae(train[TARGET].values, train.prediction.values)
 fig, ax = plt.subplots(figsize=(18,6))
plt.subplot(121)
 plt.plot(train[TARGET].values, c=RED, label='true')
 plt.plot(train.prediction.values, c=BLUE, alpha=0.5, label='prediction')
 plt.xlabel('time')
 plt.ylabel('time_to_failure')
 plt.legend()
plt.grid()
plt.subplot(122)
 plt.scatter(y=train[TARGET].values, x=feature_train, c=BLUE, alpha=0.1, label='true')
 a = np.quantile(feature_train, 0.05)
 b = np.quantile(feature_train, 0.95)
 x = np.linspace(a, b, 10)
 features = np.vstack([x] ).T
preds = model.predict(features.reshape(-1,1))
plt.plot(features, preds, c=RED, alpha=1., label='linear model')
 plt.xlabel(feature)
 plt.ylabel('time_to_failure')
 plt.legend()
plt.grid()
print("MAE =", score)
 plt.show()
return score
TARGET = 'time_to_failure'
for feature in train.columns[:-3]:
build_linear_model(feature )
out_df.columns = ["high", "medium", "low"]<feature_engineering>
feature = 'roll_100_std_percentile_5'
feature_values = train[feature].values
threshold =.32
target = train[TARGET]
index1 =(target > threshold)
 index2 =(target <= threshold)
 model1 = LinearRegression()
model1.fit(feature_values[index1].reshape(-1,1), train[TARGET].values[index1])
 model2 = LinearRegression()
model2.fit(feature_values[index2].reshape(-1,1), train[TARGET].values[index2])
 preds1 = model1.predict(feature_values.reshape(-1,1))
preds1[preds1 < 0] = 0
preds2 = model2.predict(feature_values.reshape(-1,1))
preds2[preds2 < 0] = 0
preds = model1.predict(feature_values.reshape(-1,1))
preds[index2] = preds2[index2]
preds[preds < 0] = 0
score = mae(train[TARGET].values, preds)
 print("MAE =", score)
 fig, ax = plt.subplots(figsize=(18,6))
plt.subplot(131)
 plt.plot(target, c=RED, label='true')
 plt.plot(preds, c=BLUE, alpha=0.5, label='prediction')
 plt.title('Two linear models')
 plt.xlabel('time')
 plt.ylabel('time_to_failure')
 plt.legend()
plt.grid()
plt.subplot(132)
 plt.scatter(y=target[index1], x=feature_values[index1], c=BLUE, alpha=0.1, label='true')
 a = np.quantile(feature_values, 0.05)
 b = np.quantile(feature_values, 0.95)
 x = np.linspace(a, b, 10)
 features = np.vstack([x] ).T
preds1 = model1.predict(features.reshape(-1,1))
plt.plot(features, preds1, c=RED, alpha=1., label='linear model')
 plt.title('More than {} seconds to quake'.format(threshold))
plt.xlabel(feature)
 plt.ylabel('time_to_failure')
 plt.legend()
plt.grid()
plt.subplot(133)
 plt.scatter(y=target[index2], x=feature_values[index2], c=BLUE, alpha=0.1, label='true')
 a = np.quantile(feature_values, 0.05)
 b = np.quantile(feature_values, 0.95)
 x = np.linspace(a, b, 10)
 features = np.vstack([x] ).T
preds2 = model2.predict(features.reshape(-1,1))
plt.plot(features, preds2, c=RED, alpha=1., label='linear model')
 plt.title('Less than {} seconds to quake'.format(threshold))
plt.xlabel(feature)
 plt.ylabel('time_to_failure')
 plt.legend()
plt.grid()
out_df["listing_id"] = test_df.listing_id.values
out_df<save_to_csv>
train['classifier_32'] = 0
train['target'] = train.time_to_failure < 0.32
TARGET = 'target'
features = [
'mag_freq0_percentile_50',
'bp5_mad',
'bp1_interquantile_range_5_95',
'evol_bp2_interquantile_range_10_90',
'roll_4096_mean_std',
'roll_1000_std_percentile_5',
'bp0_kurtosis',
'bp1_percentile_10',
'evol_bp3_percentile_75'
]
params = {
'num_leaves': 32,
'max_bin': 63,
'min_data_in_leaf': 50,
'learning_rate': 0.05,
'min_sum_hessian_in_leaf': 0.01,
'bagging_fraction': 0.8,
'bagging_freq': 1,
'feature_fraction': 1,
'min_gain_to_split': 0.02,
'max_depth': 6,
'save_binary': True,
'seed': 0,
'feature_fraction_seed': 0,
'bagging_seed': 0,
'drop_seed': 0,
'data_random_seed': 0,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'boost_from_average': True,
'metric': 'auc',
'is_unbalance': True
}
num_boost_round=100
for i in range(K):
val_fold = folds[i]
cv_train = train[(~np.isin(train.fold, val_fold)) ]
cv_val = train[np.isin(train.fold, val_fold)]
xg_train = lgb.Dataset(cv_train[features].values, label=cv_train[TARGET].values)
 xg_val = lgb.Dataset(cv_val[features].values, label=cv_val[TARGET].values)
 model = lgb.train(params, xg_train, num_boost_round=num_boost_round, verbose_eval=0)
 val_predictions = model.predict(cv_val[features].values, num_iteration=num_boost_round)
 train.loc[np.isin(train.fold, val_fold), 'classifier_32'] = val_predictions
xg_train = lgb.Dataset(train[features].values, label=train[TARGET].values)
 model = lgb.train(params, xg_train, num_boost_round=num_boost_round)
 test['classifier_32'] = model.predict(test[features].values, num_iteration=num_boost_round)
 fpr, tpr, _ = roc_curve(train['target'], train['classifier_32'])
 roc_auc = auc(fpr, tpr)
 plt.figure(figsize=(8,8))
lw = 2
plt.plot(fpr, tpr, color=BLUE,
lw=lw, label='ROC curve(area = %0.2f)' % roc_auc)
 plt.plot([0, 1], [0, 1], color=RED, lw=lw, linestyle='--')
 plt.xlim([0.0, 1.0])
 plt.ylim([0.0, 1.05])
 plt.xlabel('False Positive Rate')
 plt.ylabel('True Positive Rate')
 plt.title('Classifier ROC curve')
 plt.legend(loc="lower right")
 plt.show()
out_df.to_csv("xgb_starter2.csv", index=False )<load_from_disk>
features = [
'roll_2048_std_percentile_10',
'roll_4096_mean_percentile_95',
'bp6_percentile_90',
'bp2_interquantile_range_20_80',
'bp5_percentile_99',
'bp7_interquantile_range_10_90',
'bp4_percentile_80',
'roll_100_mean_mean',
'roll_1000_mean_percentile_99',
'roll_100_std_percentile_5',
'bp7_mad',
'evol_bp5_min',
'evol_bp7_max'
]
params = {'num_leaves': 5,
'max_bin': 63,
'min_data_in_leaf': 20,
'learning_rate': 0.05,
'min_sum_hessian_in_leaf': 0.008,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'feature_fraction': 1.,
'min_gain_to_split': 0.5,
'max_depth': 4,
'save_binary': True,
'seed': 0,
'feature_fraction_seed': 0,
'bagging_seed': 0,
'drop_seed': 0,
'data_random_seed': 0,
'objective': 'huber',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'mae',
'boost_from_average': True,
}
num_rounds = 2000
early_stopping = 50
TARGET = 'time_to_failure'
scores_train = []
scores_val = []
count = 0
test['lgbm_prediction'] = 0
for i in range(K):
val_fold = folds[i]
cv_train = train[(~np.isin(train.fold, val_fold)) &(train.time_to_failure > 0.32)]
cv_val = train[np.isin(train.fold, val_fold)&(train.time_to_failure > 0.0)]
xg_train = lgb.Dataset(cv_train[features].values, label=cv_train[TARGET].values)
 model = lgb.train(params, xg_train, num_rounds, verbose_eval=0)
 train_preds = model.predict(cv_train[features].values, num_iteration=num_rounds)
 val_preds = model.predict(cv_val[features].values, num_iteration=num_rounds)
 train.loc[np.isin(train.fold, val_fold), 'lgbm_prediction'] = val_preds
scores_train.append(mae(cv_train[TARGET].values, train_preds))
scores_val.append(mae(cv_val[TARGET].values, val_preds))
count += 1
test['lgbm_prediction'] += model.predict(test[features].values, num_iteration=num_rounds)
 test['lgbm_prediction'] = test['lgbm_prediction'] / count
print("MAE =", mae(train.time_to_failure[:-5], train.lgbm_prediction[:-5]))
fig, ax = plt.subplots(figsize=(18,6))
plt.subplot(111)
 plt.plot(train[TARGET].values, c=RED, label='true')
 plt.plot(train.lgbm_prediction.values, c=BLUE, alpha=0.5, label='prediction')
 plt.title('LightGBM predictions')
 plt.xlabel('time')
 plt.ylabel('time_to_failure')
 plt.legend()
plt.grid()
plt.show()
train_df=pd.read_json('.. /input/train.json')
 test_df=pd.read_json('.. /input/test.json' )<feature_engineering>
feature = 'roll_100_std_percentile_5'
scores_train = []
scores_val = []
count = 0
test['nusvr_prediction'] = 0
threshold = 0.32
for i in range(K):
val_fold = folds[i]
cv_train = train[(~np.isin(train.fold, val_fold)) &(train.time_to_failure <= threshold)]
cv_val = train[np.isin(train.fold, val_fold)&(train.time_to_failure > 0.0)]
feature_train = cv_train[feature].values
target_train = cv_train[TARGET].values
feature_val = cv_val[feature].values
target_val = cv_val[TARGET].values
feature_test = test[feature].values
params = {
'C': 30,
'gamma': 0.05,
'kernel': 'rbf',
'nu': 1,
}
model = NuSVR(**params)
 model.fit(feature_train.reshape(-1,1), 1/(threshold + 0.01 - target_train))
train_preds = threshold + 0.01 - 1/model.predict(feature_train.reshape(-1,1))
train_preds[train_preds < 0] = 0
val_preds = threshold + 0.01 - 1/model.predict(feature_val.reshape(-1,1))
val_preds[val_preds < 0] = 0
test_preds = threshold + 0.01 - 1/model.predict(feature_test.reshape(-1,1))
test_preds[test_preds < 0] = 0
train.loc[np.isin(train.fold, val_fold), 'nusvr_prediction'] = val_preds
scores_train.append(mae(cv_train[TARGET].values, train_preds))
scores_val.append(mae(cv_val[TARGET].values, val_preds))
count += 1
test['nusvr_prediction'] += test_preds
test['nusvr_prediction'] = test['nusvr_prediction'] / count
fig, ax = plt.subplots(figsize=(18,6))
plt.subplot(111)
 plt.plot(train[TARGET].values, c=RED, label='true')
 plt.plot(train.nusvr_prediction.values, c=BLUE, alpha=0.5, label='prediction')
 plt.title('NuSVR predictions')
 plt.xlabel('time')
 plt.ylabel('time_to_failure')
 plt.legend()
plt.grid()
plt.show()
train_df["price_t"] =train_df["price"]/train_df["bedrooms"]
test_df["price_t"] = test_df["price"]/test_df["bedrooms"]
train_df["room_sum"] = train_df["bedrooms"]+train_df["bathrooms"]
test_df["room_sum"] = test_df["bedrooms"]+test_df["bathrooms"]
train_df["num_photos"] = train_df["photos"].apply(len)
 test_df["num_photos"] = test_df["photos"].apply(len)
 train_df["num_features"] = train_df["features"].apply(len)
 test_df["num_features"] = test_df["features"].apply(len)
 train_df["num_description_words"] = train_df["description"].apply(lambda x: len(x.split(" ")))
 test_df["num_description_words"] = test_df["description"].apply(lambda x: len(x.split(" ")))
 features_to_use=["bathrooms", "bedrooms", "latitude", "longitude", "price","price_t","num_photos", "num_features", "num_description_words","listing_id"]<train_model>
data = []
for x in np.linspace(0.,1.,21):
def mix(row):
t = x
if row.classifier_32 < t:
return row.lgbm_prediction
else:
return row.nusvr_prediction
train['final_prediction'] = train.apply(mix, axis=1)
 data.append([np.round(x,2), mae(train.time_to_failure[:-5], train.final_prediction[:-5])])
 pd.DataFrame(data=data, columns=['threshold', 'MAE'] )
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0, num_rounds=1000):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.03
param['max_depth'] = 6
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = 1
param['subsample'] = 0.7
param['colsample_bytree'] = 0.7
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=20)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 return pred_test_y, model<categorify>
submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 submission = submission.drop('time_to_failure', axis=1)
 preds = test[['time_to_failure', 'seg_id']]
submission = submission.merge(preds, how='left')
 submission.to_csv('submission_nusvr_lgbm_classifier.csv', index=False)
 submission.head()