Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<feature_engineering>
random_seed = 4126
cast = {
'acoustic_data': 'int',
'time_to_failure': 'float'
}
def df_fragments(path, length, skip=1):
with open(path, 'r')as f:
m = {}
cols = []
count = 0
index = 0
for line in f:
if len(cols)== 0:
for col in line.strip("
\r " ).split(','):
cols.append(col)
 continue
if count == 0:
for col in cols:
m[col] = []
if index % skip == 0:
for j, cell in enumerate(line.strip("
\r " ).split(',')) :
col = cols[j]
m[col].append(getattr(builtins, cast[col] )(cell))
count += 1
if count == length:
if index % skip == 0:
yield pd.DataFrame(m)
 index += 1
count = 0
def count_rows(path):
with open(path, 'r')as f:
i = -1
for _ in f:
i += 1
return i
train_cols = df_train.columns
bin_cols = df_train.columns[df_train.columns.str.endswith('bin')] 
<data_type_conversions>
print('counting total...')
 total = count_rows('.. /input/train.csv')
 print('total: {}'.format(total))
<feature_engineering>
print('generating train data...')
 fragment_size = 150000
skip = 1
mfcc_ttf_map = collections.defaultdict(list)
 for df in tqdm(df_fragments('.. /input/train.csv', length=fragment_size, skip=skip), total=(total//fragment_size)//skip):
mfcc = librosa.feature.mfcc(df['acoustic_data'].values.astype('float32'))
mfcc_mean = mfcc.mean(axis=1)
 for i, each_mfcc_mean in enumerate(mfcc_mean):
key = 'mfcc_{}'.format(i)
 if key not in mfcc_ttf_map:
mfcc_ttf_map[key] = []
mfcc_ttf_map[key].append(each_mfcc_mean)
 key = 'time_to_failure'
if key not in mfcc_ttf_map:
mfcc_ttf_map[key] = []
mfcc_ttf_map[key].append(df.iloc[-1][key])
 mfcc_ttf_df = pd.DataFrame(mfcc_ttf_map)
 fname = 'mfcc_train.csv'
mfcc_ttf_df.to_csv(fname, index=False)
 print('saved {}.'.format(fname))
<drop_column>
def report_cv(model):
X = mfcc_ttf_df.drop(['time_to_failure'], axis=1 ).values
y = mfcc_ttf_df['time_to_failure'].values
scores = cross_val_score(model, X, y, scoring='neg_mean_absolute_error', cv=10)
 print('Cross Validation scores: {}'.format(abs(scores)))
 print('Average score: {}'.format(abs(scores.mean())))
 parameter_grid = {'n_estimators': [50, 60],
'max_depth': [5, 10]
}
report_cv(GridSearchCV(RandomForestRegressor() , param_grid=parameter_grid,n_jobs = -1))
<categorify>
report_cv(CatBoostRegressor(verbose=False))
<prepare_x_and_y><EOS> 
print('generating test features...')
 test_dir = '.. /input/test'
test_map = collections.defaultdict(list)
 for fname in tqdm(os.listdir(test_dir)) :
path = test_dir + '/' + fname
df = pd.read_csv(path)
 mfcc = librosa.feature.mfcc(df['acoustic_data'].values.astype('float32'))
mfcc_mean = mfcc.mean(axis=1)
 for i, each_mfcc_mean in enumerate(mfcc_mean):
key = 'mfcc_{}'.format(i)
 if key not in test_map:
test_map[key] = []
test_map[key].append(each_mfcc_mean)
 key = 'seg_id'
if key not in test_map:
test_map[key] = []
test_map[key].append(re.sub('.csv$', '', fname))
test_df = pd.DataFrame(test_map)
 test_csv = 'mfcc_test.csv'
test_df.to_csv(test_csv, index=False)
 print('saved {}'.format(test_csv))
def submit(model, file_path):
X = mfcc_ttf_df.drop(['time_to_failure'], axis=1 ).values
y = mfcc_ttf_df['time_to_failure'].values
model.fit(X, y)
 X_submit = test_df.drop(['seg_id'], axis=1 ).values
y_submit = model.predict(X_submit)
 submit_df = pd.DataFrame({
'seg_id': test_df['seg_id'].values,
'time_to_failure': y_submit
})
 submit_df.to_csv(file_path, index=False)
 print('saved {}'.format(file_path))