Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<concatenate>
random_seed = 4126
cast = {
'acoustic_data': 'int',
'time_to_failure': 'float'
}
def df_fragments(path, length, skip=1):
with open(path, 'r')as f:
m = {}
cols = []
count = 0
index = 0
for line in f:
if len(cols)== 0:
for col in line.strip("
\r " ).split(','):
cols.append(col)
 continue
if count == 0:
for col in cols:
m[col] = []
if index % skip == 0:
for j, cell in enumerate(line.strip("
\r " ).split(',')) :
col = cols[j]
m[col].append(getattr(builtins, cast[col] )(cell))
count += 1
if count == length:
if index % skip == 0:
yield pd.DataFrame(m)
 index += 1
count = 0
def count_rows(path):
with open(path, 'r')as f:
i = -1
for _ in f:
i += 1
return i
def fe_step1(actual):

 x, y, z, w = actual['norm_X'].tolist() , actual['norm_Y'].tolist() , actual['norm_Z'].tolist() , actual['norm_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 actual['euler_x'] = nx
actual['euler_y'] = ny
actual['euler_z'] = nz
return actual<feature_engineering>
print('counting total...')
 total = count_rows('.. /input/train.csv')
 print('total: {}'.format(total))
def feat_eng(data):
df = pd.DataFrame()
data['totl_anglr_vel'] =(data['angular_velocity_X']**2 + data['angular_velocity_Y']**2 + data['angular_velocity_Z']**2)** 0.5
data['totl_linr_acc'] =(data['linear_acceleration_X']**2 + data['linear_acceleration_Y']**2 + data['linear_acceleration_Z']**2)**0.5
data['totl_xyz'] =(data['orientation_X']**2 + data['orientation_Y']**2 + data['orientation_Z']**2)**0.5
data['acc_vs_vel'] = data['totl_linr_acc'] / data['totl_anglr_vel']
def mean_change_of_abs_change(x):
return np.mean(np.diff(np.abs(np.diff(x))))
for col in data.columns:
if col in ['row_id','series_id','measurement_number']:
continue
df[col + '_mean'] = data.groupby(['series_id'])[col].mean()
df[col + '_median'] = data.groupby(['series_id'])[col].median()
df[col + '_max'] = data.groupby(['series_id'])[col].max()
df[col + '_min'] = data.groupby(['series_id'])[col].min()
df[col + '_std'] = data.groupby(['series_id'])[col].std()
df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_maxtoMin'] = df[col + '_max'] / df[col + '_min']
df[col + '_mean_abs_chg'] = data.groupby(['series_id'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
df[col + '_mean_change_of_abs_change'] = data.groupby('series_id')[col].apply(mean_change_of_abs_change)
 df[col + '_abs_max'] = data.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 df[col + '_abs_min'] = data.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 df[col + '_abs_avg'] =(df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df<categorify>
print('generating train data...')
 fragment_size = 150000
skip = 1
mfcc_ttf_map = {}
for df in tqdm(df_fragments('.. /input/train.csv', length=fragment_size, skip=skip), total=(total//fragment_size)//skip):
mfcc = librosa.feature.mfcc(df['acoustic_data'].values.astype('float32'))
mfcc_mean = mfcc.mean(axis=1)
 for i, each_mfcc_mean in enumerate(mfcc_mean):
key = 'mfcc_{}'.format(i)
 if key not in mfcc_ttf_map:
mfcc_ttf_map[key] = []
mfcc_ttf_map[key].append(each_mfcc_mean)
 key = 'time_to_failure'
if key not in mfcc_ttf_map:
mfcc_ttf_map[key] = []
mfcc_ttf_map[key].append(df.iloc[-1][key])
 mfcc_ttf_df = pd.DataFrame(mfcc_ttf_map)
 fname = 'mfcc_train.csv'
mfcc_ttf_df.to_csv(fname, index=False)
 print('saved {}.'.format(fname))
data.fillna(0,inplace=True)
 test.fillna(0,inplace=True)
 data.replace(-np.inf,0,inplace=True)
 data.replace(np.inf,0,inplace=True)
 test.replace(-np.inf,0,inplace=True)
 test.replace(np.inf,0,inplace=True )<feature_engineering>
def report_cv(model):
X = mfcc_ttf_df.drop(['time_to_failure'], axis=1 ).values
y = mfcc_ttf_df['time_to_failure'].values
scores = cross_val_score(model, X, y, scoring='neg_mean_absolute_error', cv=10)
 print('Cross Validation scores: {}'.format(abs(scores)))
 print('Average score: {}'.format(abs(scores.mean())))
 report_cv(LinearRegression() )
target['surface'] = le.fit_transform(target['surface'] )<count_values>
report_cv(XGBRegressor(random_state=random_seed))
<define_variables><EOS> 
print('generating test features...')
 test_dir = '.. /input/test'
test_map = {}
for fname in tqdm(os.listdir(test_dir)) :
path = test_dir + '/' + fname
df = pd.read_csv(path)
 mfcc = librosa.feature.mfcc(df['acoustic_data'].values.astype('float32'))
mfcc_mean = mfcc.mean(axis=1)
 for i, each_mfcc_mean in enumerate(mfcc_mean):
key = 'mfcc_{}'.format(i)
 if key not in test_map:
test_map[key] = []
test_map[key].append(each_mfcc_mean)
 key = 'seg_id'
if key not in test_map:
test_map[key] = []
test_map[key].append(re.sub('.csv$', '', fname))
test_df = pd.DataFrame(test_map)
 test_csv = 'mfcc_test.csv'
test_df.to_csv(test_csv, index=False)
 print('saved {}'.format(test_csv))
def submit(model, file_path):
X = mfcc_ttf_df.drop(['time_to_failure'], axis=1 ).values
y = mfcc_ttf_df['time_to_failure'].values
model.fit(X, y)
 X_submit = test_df.drop(['seg_id'], axis=1 ).values
y_submit = model.predict(X_submit)
 submit_df = pd.DataFrame({
'seg_id': test_df['seg_id'].values,
'time_to_failure': y_submit
})
 submit_df.to_csv(file_path, index=False)
 print('saved {}'.format(file_path))