Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<import_modules>
warnings.filterwarnings("ignore" )
from sklearn.model_selection import KFold<define_variables>
!pip install tsfresh
n_folds = 5<train_model>
from tsfresh.feature_extraction import feature_calculators
class LGBregressor(object):
def __init__(self,params):
self.params = params
def fit(self, X, y, w):
y /= 10000000
split = int(X.shape[0] * 0.8)
 indices = np.random.permutation(X.shape[0])
 train_id, test_id = indices[:split], indices[split:]
x_train, y_train, w_train, x_valid, y_valid, w_valid = X[train_id], y[train_id], w[train_id], X[test_id], y[test_id], w[test_id],
d_train = lgb.Dataset(x_train, y_train, weight=w_train)
 d_valid = lgb.Dataset(x_valid, y_valid, weight=w_valid)
 partial_bst = lgb.train(self.params, d_train, 10000, valid_sets=d_valid, early_stopping_rounds=50, verbose_eval=500)
 num_round = partial_bst.best_iteration
d_all = lgb.Dataset(X, label = y, weight=w)
 self.bst = lgb.train(self.params, d_all, num_round)
 def predict(self, X):
return self.bst.predict(X)* 10000000 
<train_model>
!pip install tpot
class XGBregressor(object):
def __init__(self, params):
self.params = params
def fit(self, X, y, w=None):
split = int(X.shape[0] * 0.8)
 indices = np.random.permutation(X.shape[0])
 train_id, test_id = indices[:split], indices[split:]
x_train, y_train, w_train, x_valid, y_valid, w_valid = X[train_id], y[train_id], w[train_id], X[test_id], y[test_id], w[test_id],
d_train = xgb.DMatrix(x_train, label=y_train, weight=w_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid, weight=w_valid)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
partial_bst = xgb.train(self.params, d_train, 10000, early_stopping_rounds=50, evals = watchlist, verbose_eval=500)
 num_round = partial_bst.best_iteration
d_all = xgb.DMatrix(X, label = y, weight=w)
 self.bst = xgb.train(self.params, d_all, num_round)
 def predict(self, X):
test = xgb.DMatrix(X)
 return self.bst.predict(test )<prepare_x_and_y>
PATH=".. /input/"
train_df = pd.read_csv(PATH + 'train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})
 train_df.head()
class Ensemble(object):
def __init__(self, stacker, base_models):
self.stacker = stacker
self.base_models = base_models
def fit_predict(self, trainDf, testDf):
X = trainDf.drop(['price_doc', 'w'], 1 ).values
y = trainDf['price_doc'].values
w = trainDf['w'].values
T = testDf.values
X_fillna = trainDf.drop(['price_doc', 'w'], 1 ).fillna(-999 ).values
T_fillna = testDf.fillna(-999 ).values
kfold = KFold(n_splits=n_folds, shuffle=True)
 S_train = np.zeros(( X.shape[0], len(self.base_models)))
 S_test = np.zeros(( T.shape[0], len(self.base_models)))
 for i, clf in enumerate(self.base_models):
print('Training base model ' + str(i+1)+ '...')
 S_test_i = np.zeros(( T.shape[0], n_folds))
j=0
for train_idx, test_idx in kfold.split(X):
print('Training round...' + str(j+1)+ '...')
 if clf not in [xgb1,lgb1]:
X = X_fillna
T = T_fillna
X_train = X[train_idx]
y_train = y[train_idx]
w_train = w[train_idx]
X_holdout = X[test_idx]
clf.fit(X_train, y_train, w_train)
 y_pred = clf.predict(X_holdout)
 S_train[test_idx, i] = y_pred
S_test_i[:, j] = clf.predict(T)
 j=j+1
S_test[:, i] = S_test_i.mean(1)
 self.corr = pd.concat([pd.DataFrame(S_train),trainDf['price_doc']],1 ).corr()
print(S_train.shape)
 print(y.shape)
 self.stacker.fit(S_train, y, w)
 y_pred = self.stacker.predict(S_test)
 return y_pred<load_from_csv>
rows = 150000
segments = int(np.floor(train_df.shape[0] / rows))
print("Number of segments: ", segments )
trainDf = pd.read_csv('/kaggle/working/train_featured.csv')
 testDf = pd.read_csv('/kaggle/working/test_featured.csv' )<define_search_model>
train_X = pd.DataFrame(index=range(segments), dtype=np.float64)
 train_y = pd.DataFrame(index=range(segments), dtype=np.float64, columns=['time_to_failure'] )
params1 = {'eta':0.05, 'max_depth':5, 'subsample':0.8, 'colsample_bytree':0.8, 'min_child_weight':1,
'gamma':0, 'silent':1, 'objective':'reg:linear', 'eval_metric':'rmse'}
xgb1 = XGBregressor(params1 )<choose_model_class>
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def classic_sta_lta(x, length_sta, length_lta):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]
sta /= length_sta
lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]
lta /= length_lta
sta[:length_lta - 1] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta
params2 = {'booster':'gblinear', 'alpha':0,
'eta':0.1, 'max_depth':2, 'subsample':1, 'colsample_bytree':1, 'min_child_weight':1,
'gamma':0, 'silent':1, 'objective':'reg:linear', 'eval_metric':'rmse'}
xgb2 = XGBregressor(params2 )<prepare_x_and_y>
def create_features(seg_id, xc, X):
zc = np.fft.fft(xc)
 X.loc[seg_id,'cid_ce1']=feature_calculators.cid_ce(xc, 1)
 realFFT = np.real(zc)
 imagFFT = np.imag(zc)
 X.loc[seg_id, 'Rmin'] = realFFT.min()
X.loc[seg_id, 'Imin'] = imagFFT.min()
X.loc[seg_id, 'Rmax_last_15000'] = realFFT[-15000:].max()
X.loc[seg_id, 'Rmin_last_15000'] = realFFT[-15000:].min()
X.loc[seg_id, 'autocorrelation_10'] = feature_calculators.autocorrelation(xc, 10)
 X.loc[seg_id, 'Hilbert_mean'] = np.abs(hilbert(xc)).mean()
X.loc[seg_id, 'classic_sta_lta1_mean'] = classic_sta_lta(xc, 500, 10000 ).mean()
no_of_std = 2
X.loc[seg_id, 'Moving_average_700_mean'] = xc.rolling(window=700 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_400_mean'] = xc.rolling(window=400 ).mean().mean(skipna=True)
 X.loc[seg_id, 'MA_700MA_std_mean'] = xc.rolling(window=700 ).std().mean()
X.loc[seg_id,'MA_700MA_BB_high_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X.loc[seg_id,'MA_700MA_BB_low_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X.loc[seg_id, 'MA_400MA_std_mean'] = xc.rolling(window=400 ).std().mean()
X.loc[seg_id,'MA_400MA_BB_high_mean'] =(X.loc[seg_id, 'Moving_average_400_mean'] + no_of_std * X.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X.loc[seg_id,'MA_400MA_BB_low_mean'] =(X.loc[seg_id, 'Moving_average_400_mean'] - no_of_std * X.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X.loc[seg_id, 'MA_1000MA_std_mean'] = xc.rolling(window=1000 ).std().mean()
X.loc[seg_id,'kstat2']=sp.stats.kstat(xc, 2)
 X.loc[seg_id,'kstat3']=sp.stats.kstat(xc, 3)
 X.loc[seg_id,'abs_sum_changes']=feature_calculators.absolute_sum_of_changes(xc)
 X.loc[seg_id,'mean_abs_change']=feature_calculators.mean_abs_change(xc)
 X.loc[seg_id,'ratio_value_number_to_timeseries']=feature_calculators.ratio_value_number_to_time_series_length(xc)
 X.loc[seg_id,'ac10']=feature_calculators.autocorrelation(xc, 10)
 X.loc[seg_id,'ac50']=feature_calculators.autocorrelation(xc, 50)
 X.loc[seg_id,'npeaks_0']=feature_calculators.number_crossing_m(xc, 0)
 X.loc[seg_id,'npeaks_10']=feature_calculators.number_peaks(xc, 10)
 X.loc[seg_id,'npeaks_50']=feature_calculators.number_peaks(xc, 50)
 X.loc[seg_id,'npeaks_100']=feature_calculators.number_peaks(xc, 100)
 X.loc[seg_id,'lsbm']=feature_calculators.longest_strike_below_mean(xc)
 X.loc[seg_id,'lsam']=feature_calculators.longest_strike_above_mean(xc)
 X.loc[seg_id, 'MA_15000MA_std_mean'] = xc.rolling(window=15000 ).std().mean()
for windows in [10]:
x_roll_std = xc.rolling(windows ).std().dropna().values
x_roll_mean = xc.rolling(windows ).mean().dropna().values
X.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
 X.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 X.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 X.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 X.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 for windows in [100]:
X.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
 for windows in [1000]:
X.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
<choose_model_class>
RF = RandomForestRegressor(n_estimators=500, max_features=0.2)
 ETR = ExtraTreesRegressor(n_estimators=500, max_features=0.3, max_depth=None)
 Ada = AdaBoostRegressor(DecisionTreeRegressor(max_depth=15),n_estimators=200)
 GBR = GradientBoostingRegressor(n_estimators=200,max_depth=5,max_features=0.5)
 LR =LinearRegression()
params_lgb = {'objective':'regression','metric':'rmse',
'learning_rate':0.05,'max_depth':-1,'sub_feature':0.7,'sub_row':1,
'num_leaves':15,'min_data':30,'max_bin':20,
'bagging_fraction':0.9,'bagging_freq':40,'verbosity':0}
lgb1 = LGBregressor(params_lgb)
 E = Ensemble(xgb2, [xgb1,lgb1,RF,ETR,Ada,GBR] )<predict_on_test>
for seg_id in tqdm_notebook(range(segments)) :
seg = train_df.iloc[seg_id*rows:seg_id*rows + rows]
create_features(seg_id, seg['acoustic_data'], train_X)
 train_y.loc[seg_id, 'time_to_failure'] = seg['time_to_failure'].values[-1]
prediction = E.fit_predict(trainDf, testDf )<save_to_csv>
scaler = StandardScaler()
scaler.fit(train_X)
 scaled_train_X = pd.DataFrame(scaler.transform(train_X), columns=train_X.columns )
output = pd.read_csv('.. /input/test.csv')
 output = output[['id']]
output['price_doc'] = prediction
output.to_csv(r'Submission_Stack.csv',index=False)
<install_modules>
import statsmodels.api as sm
from scipy.stats.mstats import zscore
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py > /dev/null
!python pytorch-xla-env-setup.py --version 20200416 --apt-packages libomp5 libopenblas-dev > /dev/null
!pip install transformers > /dev/null
!pip install pandarallel > /dev/null<set_options>
scaled_train_X = scaled_train_X.drop(['Moving_average_700_mean'],axis=1)
 scaled_train_X = scaled_train_X.drop(['Moving_average_400_mean'],axis=1)
tqdm.pandas()
nltk.download('punkt')
 pandarallel.initialize(nb_workers=2, progress_bar=True )<define_variables>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 test_X = pd.DataFrame(columns=train_X.columns, dtype=np.float32, index=submission.index)
 for seg_id in tqdm_notebook(test_X.index):
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 create_features(seg_id, seg['acoustic_data'], test_X)
 scaled_test_X = pd.DataFrame(scaler.transform(test_X), columns=test_X.columns )
SEED = 42
MAX_LENGTH = 224
BACKBONE_PATH = '.. /input/multitpu-inference'
CHECKPOINT_PATH = '.. /input/multitpu-inference/checkpoint-xlm-roberta.bin'
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<string_transform>
scaled_test_X = scaled_test_X.drop(['Moving_average_700_mean'],axis=1)
 scaled_test_X = scaled_test_X.drop(['Moving_average_400_mean'],axis=1)
LANGS = {
'en': 'english',
'it': 'italian',
'fr': 'french',
'es': 'spanish',
'tr': 'turkish',
'ru': 'russian',
'pt': 'portuguese'
}
def get_sentences(text, lang='en'):
return sent_tokenize(text, LANGS.get(lang, 'english'))
def exclude_duplicate_sentences(text, lang='en'):
sentences = []
for sentence in get_sentences(text, lang):
sentence = sentence.strip()
if sentence not in sentences:
sentences.append(sentence)
 return ' '.join(sentences)
 def clean_text(text, lang='en'):
text = str(text)
 text = re.sub(r'[0-9"]', '', text)
 text = re.sub(r'
text = re.sub(r'@[\S]+\b', '', text)
 text = re.sub(r'https?\S+', '', text)
 text = re.sub(r'\s+', ' ', text)
 text = exclude_duplicate_sentences(text, lang)
 return text.strip()<categorify>
params = {'num_leaves': 240,
'min_data_in_leaf': 180,
'objective':'regression',
'max_depth': -1,
'learning_rate': 0.001,
"boosting": "gbdt",
"feature_fraction": 0.91,
"bagging_freq": 2,
"bagging_fraction": 0.91,
"bagging_seed": 42,
"metric": 'mae',
"lambda_l1": 0.1,
"verbosity": -1,
"random_state": 42}
oof = np.zeros(len(scaled_train_X))
predictions = np.zeros(len(scaled_test_X))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(scaled_train_X,train_y.values)) :
strLog = "fold {}".format(fold_)
 print(strLog)
 X_tr, X_val = scaled_train_X.iloc[trn_idx], scaled_train_X.iloc[val_idx]
y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]
model = lgb.LGBMRegressor(**params, n_estimators = 20000, n_jobs = -1)
 model.fit(X_tr,
y_tr,
eval_set=[(X_tr, y_tr),(X_val, y_val)],
eval_metric='mae',
verbose=1000,
early_stopping_rounds=900)
 oof[val_idx] = model.predict(X_val, num_iteration=model.best_iteration_)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = train_columns
fold_importance_df["importance"] = model.feature_importances_[:len(train_columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += model.predict(scaled_test_X, num_iteration=model.best_iteration_)/ folds.n_splits
<load_from_csv><EOS> 
submission.time_to_failure = predictions
submission.to_csv('submission.csv',index=True )