Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<train_model>
pd.options.display.precision = 15
get_ipython().run_line_magic('matplotlib', 'inline')
params = {
'objective':'reg:linear',
'n_estimators': 1000,
'booster':'gbtree',
'max_depth':4,
'eval_metric':'mae',
'learning_rate':0.05,
'min_child_weight':1,
'subsample':0.7,
'colsample_bytree':0.81,
'seed':45,
'reg_alpha':1e-05,
'gamma':0,
'nthread':-1
}
watchlist = [(d_train, 'train'),(d_valid, 'valid')]
clf = xgb.train(params, d_train, 5000, watchlist, early_stopping_rounds=300, maximize=False, verbose_eval=10)
 p_test = clf.predict(d_test )<define_variables>
train_dataset = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32} )
leaks = {
1461:105000,
1477:290941,
1492:67500,
1494:362500,
1514:84900,
1521:108538,
1531:80400,
1537:12789,
1540:76500,
1545:134000,
1554:122000,
1556:107500,
1557:100000,
1559:93369,
1560:114900,
1566:270000,
1567:85000,
1572:128000,
1573:308030,
1575:270000,
1586:84900,
1587:155891,
1589:64000,
1595:100000,
1597:215000,
1603:50138,
1610:174000,
1611:169000,
1615:76000,
1616:88250,
1617:85500,
1620:159000,
1622:161000,
1631:240000,
1638:154000,
1650:76500,
1652:111000,
1661:462000,
1664:610000,
1666:296000,
1678:552000,
1696:245000,
1698:327000,
1712:264500,
1717:152000,
1720:203000,
1726:171500,
1727:145000,
1737:275000,
1767:256000,
1774:135000,
1786:142900,
1787:156500,
1788:59000,
1790:78500,
1793:163000,
1807:103500,
1814:80000,
1820:58500,
1823:44000,
1831:179900,
1832:62500,
1835:97500,
1837:70000,
1842:63000,
1843:113500,
1863:269500,
1864:269500,
1892:85000,
1895:103500,
1912:315000,
1913:123000,
1915:230000,
1916:57625,
1925:170000,
1946:115000,
1947:334000,
1967:317500,
1970:390000,
1971:460000,
1975:615000,
1976:284000,
1996:284500,
1997:291000,
2004:297900,
2014:163000,
2030:300000,
2031:285000,
2032:290000,
2033:305000,
2038:345000,
2052:140000,
2055:141500,
2068:146000,
2076:94000,
2086:143000,
2093:122250,
2099:46500,
2100:65000,
2101:139500,
2106:55000,
2107:184000,
2111:108000,
2152:260000,
2162:475000,
2163:395039,
2180:185000,
2185:165000,
2206:104000,
2207:257076,
2208:263400,
2211:126000,
2217:13100,
2220:65000,
2223:300000,
2227:241500,
2230:172500,
2232:150000,
2235:195000,
2236:298751,
2238:209200,
2239:146000,
2245:94900,
2251:103000,
2263:349265,
2264:591587,
2267:441929,
2268:455000,
2269:174000,
2288:322400,
2295:500067,
2342:260000,
2354:146000,
2362:300000,
2375:279700,
2376:255000,
2379:240050,
2380:162500,
2395:224500,
2404:175000,
2419:115000,
2437:125500,
2455:136500,
2461:132000,
2465:165000,
2466:90000,
2468:113000,
2469:117000,
2474:50000,
2495:109900,
2544:110000,
2550:183850,
2557:79275,
2564:238000,
2565:153500,
2572:200000,
2574:315000,
2583:375000,
2590:244000,
2591:257000,
2599:392000,
2610:138000,
2611:80000,
2617:169000,
2618:252000,
2627:130000,
2631:535000,
2632:401179,
2634:470000,
2638:294323,
2658:344133,
2673:246990,
2690:405749,
2702:129500,
2723:157500,
2741:132000,
2752:167000,
2754:180000,
2760:80000,
2775:111500,
2776:156500,
2779:111500,
2788:64000,
2793:202500,
2794:75000,
2805:125000,
2813:156500,
2823:415000,
2829:224500,
2832:233555,
2859:98000,
2866:134000,
2872:35000,
2873:121000,
2881:195000,
2916:71000,
2917:131000,
2919:188000
}<save_to_csv>
instances = 150000
segments = int(np.floor(train_dataset.shape[0] / instances))
X_trainset = pd.DataFrame(index=range(segments), dtype=np.float64)
 y_trainset = pd.DataFrame(index=range(segments), dtype=np.float64,
columns=['time_to_failure'])
 for segment in tqdm_notebook(range(segments)) :
each_seg = train_dataset.iloc[segment * instances : segment * instances + instances]
x_rawdata = each_seg['acoustic_data']
x = x_rawdata.values
y = each_seg['time_to_failure'].values[-1]
y_trainset.loc[segment, 'time_to_failure'] = y
X_trainset.loc[segment, 'average'] = x.mean()
X_trainset.loc[segment, 'standard_deviation'] = x.std()
X_trainset.loc[segment, 'maximum'] = x.max()
X_trainset.loc[segment, 'minimum'] = x.min()
X_trainset.loc[segment, 'quantile_1_percentile'] = np.quantile(x,0.01)
 X_trainset.loc[segment, 'quantile_5_percentile'] = np.quantile(x,0.05)
 X_trainset.loc[segment, 'quantile_95_percentile'] = np.quantile(x,0.95)
 X_trainset.loc[segment, 'quantile_99_percentile'] = np.quantile(x,0.99)
 X_trainset.loc[segment, 'median_absolute'] = np.median(np.abs(x))
X_trainset.loc[segment, 'quantile_95_percentile_absolute'] = np.quantile(np.abs(x),0.95)
 X_trainset.loc[segment, 'quantile_99_percentile_absolute'] = np.quantile(np.abs(x),0.99)
 X_trainset.loc[segment, 'F_test_measure'], X_trainset.loc[segment, 'p_test_measure'] = stats.f_oneway(x[:30000],x[30000:60000],x[60000:90000],x[90000:120000],x[120000:])
 X_trainset.loc[segment, 'average_change_absolute'] = np.mean(np.diff(x))
X_trainset.loc[segment, 'average_change_rate'] = np.mean(np.nonzero(( np.diff(x)/ x[:-1])) [0])
 X_trainset.loc[segment, 'maximum_absolute'] = np.abs(x ).max()
for windows in [10,100]:
x_roll_std = x_rawdata.rolling(windows ).std().dropna().values
x_roll_mean = x_rawdata.rolling(windows ).mean().dropna().values
X_trainset.loc[segment, 'average_rolling_standard_deviation' + str(windows)] = x_roll_std.mean()
X_trainset.loc[segment, 'standard_deviation_rolling_standard_deviation' + str(windows)] = x_roll_std.std()
X_trainset.loc[segment, 'maximum_rolling_standard_deviation' + str(windows)] = x_roll_std.max()
X_trainset.loc[segment, 'minimum_rolling_standard_deviation' + str(windows)] = x_roll_std.min()
X_trainset.loc[segment, 'quantile_1_percentile_rolling_standard_deviation' + str(windows)] = np.quantile(x_roll_std,0.01)
 X_trainset.loc[segment, 'quantile_5_percentile_rolling_standard_deviation' + str(windows)] = np.quantile(x_roll_std,0.05)
 X_trainset.loc[segment, 'quantile_95_percentile_rolling_standard_deviation' + str(windows)] = np.quantile(x_roll_std,0.95)
 X_trainset.loc[segment, 'quantile_99_percentile_rolling_standard_deviation' + str(windows)] = np.quantile(x_roll_std,0.99)
 X_trainset.loc[segment, 'average_change_absolute_rolling_standard_deviation' + str(windows)] = np.mean(np.diff(x_roll_std))
X_trainset.loc[segment, 'average_change_rate_rolling_standard_deviation' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X_trainset.loc[segment, 'maximum_absolute_rolling_standard_deviation' + str(windows)] = np.abs(x_roll_std ).max()
X_trainset.loc[segment, 'average_absolute_rolling_mean' + str(windows)] = x_roll_mean.mean()
X_trainset.loc[segment, 'standard_deviation_rolling_mean' + str(windows)] = x_roll_mean.std()
X_trainset.loc[segment, 'maximum_rolling_mean' + str(windows)] = x_roll_mean.max()
X_trainset.loc[segment, 'minimum_rolling_mean' + str(windows)] = x_roll_mean.min()
X_trainset.loc[segment, 'quantile_1_percentile_rolling_mean' + str(windows)] = np.quantile(x_roll_mean,0.01)
 X_trainset.loc[segment, 'quantile_5_percentile_rolling_mean' + str(windows)] = np.quantile(x_roll_mean,0.05)
 X_trainset.loc[segment, 'quantile_95_percentile_rolling_mean' + str(windows)] = np.quantile(x_roll_mean,0.95)
 X_trainset.loc[segment, 'quantile_99_percentile_rolling_mean' + str(windows)] = np.quantile(x_roll_mean,0.99)
 X_trainset.loc[segment, 'average_change_absolute_rolling_mean' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_trainset.loc[segment, 'average_change_rate_rolling_mean' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X_trainset.loc[segment, 'maximum_absolute_rolling_mean' + str(windows)] = np.abs(x_roll_mean ).max()
sub = pd.DataFrame()
sub['Id'] = test_id
sub['SalePrice'] = p_test
sub['SalePrice'] = sub.apply(lambda r: leaks[int(r['Id'])] if int(r['Id'])in leaks else r['SalePrice'], axis=1)
 sub.to_csv('submission.csv', index=False )<import_modules>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 X_testset = pd.DataFrame(columns=X_trainset.columns, dtype=np.float64, index=submission.index)
 for i, seg_id in enumerate(tqdm_notebook(X_testset.index)) :
each_seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 x_rawdata = each_seg['acoustic_data']
x_roll = x_rawdata.rolling(windows ).std().dropna().values
x = x_rawdata.values
X_testset.loc[seg_id, 'average'] = x.mean()
X_testset.loc[seg_id, 'standard_deviation'] = x.std()
X_testset.loc[seg_id, 'maximum'] = x.max()
X_testset.loc[seg_id, 'minimum'] = x.min()
X_testset.loc[seg_id, 'quantile_1_percentile'] = np.quantile(x,0.01)
 X_testset.loc[seg_id, 'quantile_5_percentile'] = np.quantile(x,0.05)
 X_testset.loc[seg_id, 'quantile_95_percentile'] = np.quantile(x,0.95)
 X_testset.loc[seg_id, 'quantile_99_percentile'] = np.quantile(x,0.99)
 X_testset.loc[seg_id, 'median_absolute'] = np.median(np.abs(x))
X_testset.loc[seg_id, 'quantile_95_percentile_absolute'] = np.quantile(np.abs(x),0.95)
 X_testset.loc[seg_id, 'quantile_99_percentile_absolute'] = np.quantile(np.abs(x),0.99)
 X_testset.loc[seg_id, 'F_test_measure'], X_trainset.loc[segment, 'p_test_measure'] = stats.f_oneway(x[:30000],x[30000:60000],x[60000:90000],x[90000:120000],x[120000:])
 X_testset.loc[seg_id, 'average_change_absolute'] = np.mean(np.diff(x))
X_testset.loc[seg_id, 'average_change_rate'] = np.mean(np.nonzero(( np.diff(x)/ x[:-1])) [0])
 X_testset.loc[seg_id, 'maximum_absolute'] = np.abs(x ).max()
for windows in [10,100]:
x_roll_std = x_rawdata.rolling(windows ).std().dropna().values
x_roll_mean = x_rawdata.rolling(windows ).mean().dropna().values
X_testset.loc[seg_id, 'average_rolling_standard_deviation' + str(windows)] = x_roll_std.mean()
X_testset.loc[seg_id, 'standard_deviation_rolling_standard_deviation' + str(windows)] = x_roll_std.std()
X_testset.loc[seg_id, 'maximum_rolling_standard_deviation' + str(windows)] = x_roll_std.max()
X_testset.loc[seg_id, 'minimum_rolling_standard_deviation' + str(windows)] = x_roll_std.min()
X_testset.loc[seg_id, 'quantile_1_percentile_rolling_standard_deviation' + str(windows)] = np.quantile(x_roll_std,0.01)
 X_testset.loc[seg_id, 'quantile_5_percentile_rolling_standard_deviation' + str(windows)] = np.quantile(x_roll_std,0.05)
 X_testset.loc[seg_id, 'quantile_95_percentile_rolling_standard_deviation' + str(windows)] = np.quantile(x_roll_std,0.95)
 X_testset.loc[seg_id, 'quantile_99_percentile_rolling_standard_deviation' + str(windows)] = np.quantile(x_roll_std,0.99)
 X_testset.loc[seg_id, 'average_change_absolute_rolling_standard_deviation' + str(windows)] = np.mean(np.diff(x_roll_std))
X_testset.loc[seg_id, 'average_change_rate_rolling_standard_deviation' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X_testset.loc[seg_id, 'maximum_absolute_rolling_standard_deviation' + str(windows)] = np.abs(x_roll_std ).max()
X_testset.loc[seg_id, 'average_absolute_rolling_mean' + str(windows)] = x_roll_mean.mean()
X_testset.loc[seg_id, 'standard_deviation_rolling_mean' + str(windows)] = x_roll_mean.std()
X_testset.loc[seg_id, 'maximum_rolling_mean' + str(windows)] = x_roll_mean.max()
X_testset.loc[seg_id, 'minimum_rolling_mean' + str(windows)] = x_roll_mean.min()
X_testset.loc[seg_id, 'quantile_1_percentile_rolling_mean' + str(windows)] = np.quantile(x_roll_mean,0.01)
 X_testset.loc[seg_id, 'quantile_5_percentile_rolling_mean' + str(windows)] = np.quantile(x_roll_mean,0.05)
 X_testset.loc[seg_id, 'quantile_95_percentile_rolling_mean' + str(windows)] = np.quantile(x_roll_mean,0.95)
 X_testset.loc[seg_id, 'quantile_99_percentile_rolling_mean' + str(windows)] = np.quantile(x_roll_mean,0.99)
 X_testset.loc[seg_id, 'average_change_absolute_rolling_mean' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_testset.loc[seg_id, 'average_change_rate_rolling_mean' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X_testset.loc[seg_id, 'maximum_absolute_rolling_mean' + str(windows)] = np.abs(x_roll_mean ).max()
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib
import matplotlib.pyplot as plt
from scipy.stats import skew<load_from_csv>
num_folds = 5
k_folds = KFold(n_splits=num_folds, shuffle=True, random_state=11 )
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )<define_variables>
def train_model_lgb(X=X_trainset, X_testset=X_testset, y=y_trainset, params=None, k_folds=k_folds, model=None):
x_values = np.zeros(len(X))
prediction = np.zeros(len(X_testset))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(trainset_index, valid_set_index)in enumerate(k_folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train_per_fold, X_valid_per_fold = X.iloc[trainset_index], X.iloc[valid_set_index]
y_train_per_fold, y_valid_per_fold = y.iloc[trainset_index], y.iloc[valid_set_index]
model = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model.fit(X_train_per_fold, y_train_per_fold,
eval_set=[(X_train_per_fold, y_train_per_fold),(X_valid_per_fold, y_valid_per_fold)], eval_metric='mae',
verbose=1000, early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid_per_fold)
 y_pred = model.predict(X_testset, num_iteration=model.best_iteration_)
 x_values[valid_set_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_absolute_error(y_valid_per_fold, y_pred_valid))
prediction += y_pred
prediction /= num_folds
print('CV mean score: {0:.4f}.'.format(mean_absolute_error(y, x_values)))
 return x_values, prediction
def train_model_xgb(X=X_trainset, X_testset=X_testset, y=y_trainset, params=None, k_folds=k_folds, model=None):
x_value = np.zeros(len(X))
prediction = np.zeros(len(X_testset))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(trainset_index, valid_set_index)in enumerate(k_folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train_per_fold, X_valid_per_fold = X.iloc[trainset_index], X.iloc[valid_set_index]
y_train_per_fold, y_valid_per_fold = y.iloc[trainset_index], y.iloc[valid_set_index]
train_data = xgb.DMatrix(data=X_train_per_fold, label=y_train_per_fold, feature_names=X_trainset.columns)
 valid_data = xgb.DMatrix(data=X_valid_per_fold, label=y_valid_per_fold, feature_names=X_trainset.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid_per_fold, feature_names=X_trainset.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_testset, feature_names=X_trainset.columns), ntree_limit=model.best_ntree_limit)
 x_value[valid_set_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_absolute_error(y_valid_per_fold, y_pred_valid))
prediction += y_pred
prediction /= num_folds
print('CV mean score: {0:.4f}.'.format(mean_absolute_error(y, x_value)))
 return x_value, prediction
features = ['GrLivArea','OverallQual','2ndFlrSF','TotalBsmtSF','1stFlrSF','GarageCars','YearBuilt', 'GarageArea','ExterQual',
'YearRemodAdd','FireplaceQu','GarageYrBlt','FullBath','MSSubClass','LotArea','Fireplaces','KitchenQual','MSZoning',
'GarageType','BsmtFinSF1','Neighborhood','BsmtQual','TotRmsAbvGrd','HalfBath','BldgType','GarageFinish',
'Foundation','BedroomAbvGr','HouseStyle','CentralAir','OpenPorchSF','HeatingQC','BsmtFinType1','BsmtUnfSF',
'GarageCond','GarageQual','KitchenAbvGr','OverallCond','BsmtCond','MasVnrArea','BsmtFullBath','Exterior1st',
'Exterior2nd','PavedDrive','WoodDeckSF','LandContour','MasVnrType','BsmtFinType2','Functional','Fence']<drop_column>
lgb_params = {'num_leaves': 64,
'min_data_in_leaf': 50,
'objective': 'mae',
'max_depth': -1,
'learning_rate': 0.001,
"boosting": "gbdt",
"feature_fraction": 0.5,
"bagging_freq": 2,
"bagging_fraction": 0.5,
"bagging_seed": 0,
"metric": 'mae',
"verbosity": -1,
'reg_alpha': 1.0,
'reg_lambda': 1.0,
}
x_value_lgb, prediction_lgb = train_model_lgb(params = lgb_params )
train = train[features + ['SalePrice']]
test = test[['Id'] + features]<concatenate>
xgb_params = {'eta': 0.01,
'max_depth': 6,
'subsample': 0.8,
'colsample_bytree': 0.8,
'colsample_bylevel': 0.8,
'colsample_bynode': 0.8,
'lambda': 0.1,
'alpha' : 0.1,
'objective': 'reg:linear',
'eval_metric': 'mae',
'silent': True,
'nthread': 4}
x_value_xgb, prediction_xgb = train_model_xgb(params=xgb_params )
all_data = pd.concat(( train.loc[:, 'GrLivArea':'Fence'], test.loc[:,'GrLivArea':'Fence']))<create_dataframe>
print(mean_absolute_error(y_trainset,(x_value_lgb)) )
matplotlib.rcParams['figure.figsize'] =(17.0, 7.0)
 prices = pd.DataFrame({"price": train['SalePrice'], "log(price+1)": np.log1p(train['SalePrice'])})
 prices.hist() ;<feature_engineering>
submission['time_to_failure'] =(prediction_lgb)
 print(submission.head())
 submission.to_csv('submission_all.csv' )