Kaggle competition: LANL Earthquake Prediction
train.dtypes.value_counts()<set_options>
train_df=pd.read_csv('.. /input/train.csv',iterator=True,chunksize=150_000, dtype={'acoustic_data':np.int16,'time_to_failure':np.float32} )
sns.set_style("darkgrid" )<sort_values>
submission=pd.read_csv('.. /input/sample_submission.csv')
 print("No.of rows in submission :{}".format(submission.shape[0]))
train.sort_values('GrLivArea', ascending=False ).head(2)['GrLivArea']<filter>
X_train = pd.DataFrame()
y_train = pd.Series()
for df in train_df:
ch = gen_features(df['acoustic_data'])
 X_train = X_train.append(ch, ignore_index=True)
 y_train = y_train.append(pd.Series(df['time_to_failure'].values[-1]))
train = train[train['GrLivArea'] < 4676]<sort_values>
X_train.columns=['ave', 'std', 'max', 'min','kurtosis','skew','1%_quantile','5%_quantile','95%_quantile',
'99%_quantile','abs_max','abs_mean','abs_std']
train.sort_values('GrLivArea', ascending=False ).head(2)['GrLivArea']<feature_engineering>
scaler = StandardScaler()
scaler.fit(X_train)
 X_train_scaled = scaler.transform(X_train )
train['SalePrice'] = np.log1p(train['SalePrice'] )<concatenate>
X_trai, X_tes, y_trai, y_tes = train_test_split(X_train_scaled, y_train, test_size=0.2, random_state=123)
 xg_reg = xgb.XGBRegressor(objective='reg:linear',n_estimators=150,seed=123)
 xg_reg.fit(X_trai,y_trai)
 preds = xg_reg.predict(X_tes)
 mae =(mean_absolute_error(y_tes, preds))
print("MAE: %f" %(mae))
test_dummy = test.copy()
test_dummy['SalePrice'] = np.zeros(len(test))
combined = pd.concat([train,test_dummy] )<count_missing_values>
dmatrix = xgb.DMatrix(data=X_train_scaled, label=y_train)
 reg_params = [0.05, 0.0001, 0.0005]
params = {"objective":"reg:linear","max_depth":3}
mae_l2 = []
for reg in reg_params:
params["lambda"] = reg
cv_results_mae = xgb.cv(dtrain=dmatrix, params=params, nfold=10, num_boost_round=100, metrics="mae", as_pandas=True, seed=123)
 mae_l2.append(cv_results_mae["test-mae-mean"].tail(1 ).values[0])
 print("Best mae as a function of l2:")
 print(pd.DataFrame(list(zip(reg_params, mae_l2)) , columns=["l2","mae"]))
def missing_percent(data, n=35):
num_of_nulls = data.isnull().sum().sort_values(ascending=False)
 percent =(data.isnull().sum() /data.isnull().count() ).sort_values(ascending=False)
 result = pd.concat([num_of_nulls, percent], axis=1, keys=['Number', 'Percent'])
 return result.head(n )<count_missing_values>
train_pool = Pool(X_train, y_train)
 m = CatBoostRegressor(iterations=10000, loss_function='MAE', boosting_type='Ordered')
 m.fit(X_train_scaled, y_train, silent=True)
 m.best_score_
missing_percent(test )<set_options>
warnings.filterwarnings("ignore" )<categorify>
def fill_missing(df):
df['Functional'] = df['Functional'].fillna('Typ')
 cats = ['Electrical','SaleType','Exterior2nd','KitchenQual','Exterior1st','MSZoning']
for cat in cats:
df[cat] = df[cat].fillna(df[cat].mode() [0])
 df['LotFrontage'] = df['LotFrontage'].fillna(
 df.groupby('Neighborhood')['LotFrontage'].transform('median'))
df_cat = df[list(df.select_dtypes(include='object' ).columns)]
df.update(df_cat.fillna('None'))
df_num = df[list(df.select_dtypes(include='number' ).columns)]
df.update(df_num.fillna(0))
return df<count_missing_values>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id' )
fill_missing(train ).isnull().sum().sum()<count_missing_values>
X_test = pd.DataFrame(columns=X_train.columns, dtype=np.float64, index=submission.index )
fill_missing(test ).isnull().sum().sum()<concatenate>
for seg_id in X_test.index:
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 x = seg['acoustic_data']
X_test.loc[seg_id, 'ave'] = x.mean()
X_test.loc[seg_id, 'std'] = x.std()
X_test.loc[seg_id, 'max'] = x.max()
X_test.loc[seg_id, 'min'] = x.min()
X_test.loc[seg_id, 'kurtosis'] = x.kurtosis()
X_test.loc[seg_id, 'skew'] = x.skew()
X_test.loc[seg_id, '1%_quantile'] = np.quantile(x,0.01)
 X_test.loc[seg_id, '5%_quantile'] = np.quantile(x,0.05)
 X_test.loc[seg_id, '95%_quantile'] = np.quantile(x,0.95)
 X_test.loc[seg_id, '99%_quantile'] = np.quantile(x,0.99)
 X_test.loc[seg_id, 'abs_max'] = np.abs(x ).max()
X_test.loc[seg_id, 'abs_mean'] = np.abs(x ).mean()
X_test.loc[seg_id, 'abs_std'] = np.abs(x ).std()
<count_missing_values><EOS> 
X_test_scaled = scaler.transform(X_test)
 submission['time_to_failure'] = m.predict(X_test_scaled)
 submission.to_csv('submission.csv' )